├── .gitignore ├── NOTICE ├── src └── main │ └── config │ └── checkstyle │ ├── header.txt │ └── suppressions.xml ├── phoenix5-spark └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ ├── scala │ │ └── org │ │ │ └── apache │ │ │ ├── phoenix │ │ │ └── spark │ │ │ │ ├── DefaultSource.scala │ │ │ │ ├── package.scala │ │ │ │ ├── SparkSqlContextFunctions.scala │ │ │ │ ├── DataFrameFunctions.scala │ │ │ │ └── SparkContextFunctions.scala │ │ │ └── spark │ │ │ └── sql │ │ │ └── execution │ │ │ └── datasources │ │ │ └── jdbc │ │ │ └── PhoenixJdbcDialect.scala │ └── java │ │ └── org │ │ └── apache │ │ └── phoenix │ │ └── spark │ │ └── datasource │ │ └── v2 │ │ ├── writer │ │ └── PhoenixDataWriterFactory.java │ │ └── reader │ │ ├── PhoenixInputPartition.java │ │ └── PhoenixDataSourceReadOptions.java │ ├── it │ ├── resources │ │ ├── transactionTableSetup.sql │ │ ├── tenantSetup.sql │ │ └── log4j.xml │ └── java │ │ └── org │ │ └── apache │ │ └── phoenix │ │ └── spark │ │ └── SaltedTableIT.java │ └── test │ ├── java │ └── org │ │ └── apache │ │ └── phoenix │ │ └── spark │ │ └── datasource │ │ └── v2 │ │ ├── writer │ │ ├── PhoenixTestingDataWriterFactory.java │ │ ├── PhoenixTestingWriterCommitMessage.java │ │ ├── PhoenixTestingDataWriter.java │ │ └── PhoenixTestingDataSourceWriter.java │ │ ├── reader │ │ ├── PhoenixTestingDataSourceReader.java │ │ ├── PhoenixTestingInputPartition.java │ │ └── PhoenixTestingInputPartitionReader.java │ │ └── PhoenixTestingDataSource.java │ └── resources │ └── log4j.xml ├── phoenix5-spark3 └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── org.apache.spark.sql.sources.DataSourceRegister │ ├── scala │ │ └── org │ │ │ └── apache │ │ │ ├── phoenix │ │ │ └── spark │ │ │ │ ├── DefaultSource.scala │ │ │ │ ├── package.scala │ │ │ │ ├── SparkSqlContextFunctions.scala │ │ │ │ └── DataFrameFunctions.scala │ │ │ └── spark │ │ │ └── sql │ │ │ └── execution │ │ │ └── datasources │ │ │ └── jdbc │ │ │ └── PhoenixJdbcDialect.scala │ └── java │ │ └── org │ │ └── apache │ │ └── phoenix │ │ └── spark │ │ └── sql │ │ └── connector │ │ ├── reader │ │ ├── PhoenixInputPartition.java │ │ └── PhoenixPartitionReadFactory.java │ │ └── writer │ │ └── PhoenixDataWriterFactory.java │ └── it │ ├── resources │ ├── transactionTableSetup.sql │ ├── tenantSetup.sql │ └── log4j.xml │ └── java │ └── org │ └── apache │ └── phoenix │ └── spark │ ├── sql │ └── connector │ │ ├── reader │ │ ├── PhoenixTestScanBuilder.java │ │ ├── PhoenixTestScan.java │ │ ├── PhoenixTestPartitionReadFactory.java │ │ └── PhoenixTestPartitionReader.java │ │ ├── writer │ │ ├── PhoenixTestWriteBuilder.java │ │ ├── PhoenixTestDataWriterFactory.java │ │ ├── PhoenixTestingWriterCommitMessage.java │ │ ├── PhoenixTestDataWriter.java │ │ └── PhoenixTestBatchWrite.java │ │ └── PhoenixTestingTable.java │ └── SaltedTableIT.java ├── phoenix5-hive └── src │ ├── it │ ├── resources │ │ ├── hbase-site.xml │ │ └── log4j.properties │ └── java │ │ └── org │ │ └── apache │ │ ├── phoenix │ │ └── hive │ │ │ ├── HiveTezIT.java │ │ │ ├── HiveMapReduceIT.java │ │ │ └── HiveTestUtil.java │ │ └── hadoop │ │ └── hive │ │ └── ql │ │ ├── security │ │ └── DummyAuthenticator.java │ │ └── QTestProcessExecResult.java │ └── main │ └── java │ └── org │ └── apache │ └── phoenix │ └── hive │ ├── objectinspector │ ├── PhoenixDateObjectInspector.java │ ├── PhoenixTimestampObjectInspector.java │ ├── PhoenixLongObjectInspector.java │ ├── PhoenixBooleanObjectInspector.java │ ├── PhoenixShortObjectInspector.java │ ├── PhoenixBinaryObjectInspector.java │ ├── PhoenixByteObjectInspector.java │ ├── PhoenixFloatObjectInspector.java │ ├── PhoenixDoubleObjectInspector.java │ ├── PhoenixIntObjectInspector.java │ ├── PhoenixCharObjectInspector.java │ ├── AbstractPhoenixObjectInspector.java │ └── PhoenixDateObjectInspectorBase.java │ ├── ql │ └── pushdown │ │ └── PredicateAnalyzerFactory.java │ ├── PhoenixRowKey.java │ └── PhoenixRow.java ├── phoenix5-hive4 └── src │ ├── it │ ├── resources │ │ ├── hbase-site.xml │ │ └── log4j.properties │ └── java │ │ └── org │ │ └── apache │ │ ├── hadoop │ │ └── hive │ │ │ └── ql │ │ │ ├── dataset │ │ │ ├── Dataset.java │ │ │ └── DatasetCollection.java │ │ │ ├── qoption │ │ │ ├── QTestOptionHandler.java │ │ │ ├── QTestDisabledHandler.java │ │ │ ├── QTestTransactional.java │ │ │ ├── QTestAuthorizerHandler.java │ │ │ ├── QTestSysDbHandler.java │ │ │ └── QTestTimezoneHandler.java │ │ │ ├── scheduled │ │ │ └── QTestScheduledQueryServiceProvider.java │ │ │ ├── security │ │ │ └── DummyAuthenticator.java │ │ │ └── QTestProcessExecResult.java │ │ └── phoenix │ │ └── hive │ │ ├── HiveTezIT.java │ │ └── HiveMapReduceIT.java │ └── main │ └── java │ └── org │ └── apache │ └── phoenix │ └── hive │ ├── objectinspector │ ├── PhoenixDateObjectInspector.java │ ├── PhoenixTimestampObjectInspector.java │ ├── PhoenixLongObjectInspector.java │ ├── PhoenixBooleanObjectInspector.java │ ├── PhoenixShortObjectInspector.java │ ├── PhoenixByteObjectInspector.java │ ├── PhoenixBinaryObjectInspector.java │ ├── PhoenixFloatObjectInspector.java │ ├── PhoenixDoubleObjectInspector.java │ ├── PhoenixIntObjectInspector.java │ ├── PhoenixCharObjectInspector.java │ ├── AbstractPhoenixObjectInspector.java │ └── PhoenixDateObjectInspectorBase.java │ ├── ql │ └── pushdown │ │ └── PredicateAnalyzerFactory.java │ ├── PhoenixRowKey.java │ └── PhoenixRow.java ├── dev ├── docker │ ├── Dockerfile │ └── Dockerfile.yetus ├── rebuild_hbase.sh └── gather_machine_environment.sh ├── .github └── workflows │ └── maven.yml ├── README.md ├── .asf.yaml ├── phoenix5-connectors-assembly └── src │ └── build │ ├── package-phoenix5-connectors-to-tar-unshaded.xml │ ├── package-phoenix5-connectors-to-tar-all.xml │ └── components │ ├── phoenix5-jars.xml │ └── phoenix5-shaded-jars.xml └── BUILDING.md /.gitignore: -------------------------------------------------------------------------------- 1 | #general java 2 | *.class 3 | *.war 4 | *.jar 5 | 6 | # python 7 | *.pyc 8 | .checkstyle 9 | 10 | # eclipse stuffs 11 | .settings/* 12 | **/.settings/ 13 | .classpath 14 | .project 15 | */.externalToolBuilders 16 | */maven-eclipse.xml 17 | 18 | # intellij stuff 19 | .idea/ 20 | *.iml 21 | *.ipr 22 | *.iws 23 | 24 | #maven stuffs 25 | target/ 26 | release/ 27 | RESULTS/ 28 | CSV_EXPORT/ 29 | .DS_Store 30 | 31 | #test files 32 | derby.log -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache Phoenix 2 | Copyright 2013-2016 The Apache Software Foundation 3 | 4 | This product includes software developed by The Apache Software 5 | Foundation (http://www.apache.org/). 6 | 7 | This also includes: 8 | 9 | The phoenix-spark module has been adapted from the phoenix-spark library 10 | distributed under the terms of the Apache 2 license. Original source copyright: 11 | Copyright 2014 Simply Measured, Inc. 12 | Copyright 2015 Interset Software Inc. 13 | -------------------------------------------------------------------------------- /src/main/config/checkstyle/header.txt: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to you under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | org.apache.phoenix.spark.datasource.v2.PhoenixDataSource 19 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | org.apache.phoenix.spark.sql.connector.PhoenixDataSource 19 | -------------------------------------------------------------------------------- /phoenix5-spark/src/it/resources/transactionTableSetup.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | CREATE TABLE table5 (id BIGINT NOT NULL PRIMARY KEY, table5_id BIGINT, "t5col1" VARCHAR) TRANSACTIONAL=true -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/resources/transactionTableSetup.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | CREATE TABLE table5 (id BIGINT NOT NULL PRIMARY KEY, table5_id BIGINT, "t5col1" VARCHAR) TRANSACTIONAL=true -------------------------------------------------------------------------------- /phoenix5-spark/src/main/scala/org/apache/phoenix/spark/DefaultSource.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark 19 | 20 | import org.apache.phoenix.spark.datasource.v2.PhoenixDataSource 21 | 22 | class DefaultSource extends PhoenixDataSource 23 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/scala/org/apache/phoenix/spark/DefaultSource.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark 19 | 20 | import org.apache.phoenix.spark.sql.connector.PhoenixDataSource 21 | 22 | class DefaultSource extends PhoenixDataSource 23 | -------------------------------------------------------------------------------- /phoenix5-hive/src/it/resources/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 17 | 18 | 19 | 20 | hbase.wal.provider 21 | filesystem 22 | 23 | 24 | hbase.wal.meta_provider 25 | filesystem 26 | 27 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/resources/hbase-site.xml: -------------------------------------------------------------------------------- 1 | 17 | 18 | 19 | 20 | hbase.wal.provider 21 | filesystem 22 | 23 | 24 | hbase.wal.meta_provider 25 | filesystem 26 | 27 | -------------------------------------------------------------------------------- /dev/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | FROM apache/yetus:0.13.0 18 | 19 | RUN groupadd --non-unique -g 910 jenkins || true 20 | RUN useradd -g 910 -u 910 -m jenkins || true 21 | ##### 22 | # /run/user directory for gpg 23 | ##### 24 | RUN mkdir -p /run/user/910/gnupg/ \ 25 | && chmod -R 700 /run/user/910 \ 26 | && chown -R 910:910 /run/user/910 \ 27 | && mkdir -p /var/run/user/910/gnupg \ 28 | && chown -R 910:910 /var/run/user/910 \ 29 | && chmod -R 700 /var/run/user/910 -------------------------------------------------------------------------------- /phoenix5-spark/src/it/resources/tenantSetup.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | CREATE VIEW IF NOT EXISTS TENANT_VIEW(TENANT_ONLY_COL VARCHAR) AS SELECT * FROM MULTITENANT_TEST_TABLE 18 | UPSERT INTO TENANT_VIEW (ORGANIZATION_ID, TENANT_ONLY_COL) VALUES ('defaultOrg', 'defaultData') 19 | CREATE VIEW IF NOT EXISTS TENANT_VIEW_WITH_SALT(TENANT_ONLY_COL VARCHAR) AS SELECT * FROM MULTITENANT_TEST_TABLE_WITH_SALT 20 | UPSERT INTO TENANT_VIEW_WITH_SALT (ORGANIZATION_ID, TENANT_ONLY_COL) VALUES ('defaultOrg', 'defaultData') 21 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/resources/tenantSetup.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | 17 | CREATE VIEW IF NOT EXISTS TENANT_VIEW(TENANT_ONLY_COL VARCHAR) AS SELECT * FROM MULTITENANT_TEST_TABLE 18 | UPSERT INTO TENANT_VIEW (ORGANIZATION_ID, TENANT_ONLY_COL) VALUES ('defaultOrg', 'defaultData') 19 | CREATE VIEW IF NOT EXISTS TENANT_VIEW_WITH_SALT(TENANT_ONLY_COL VARCHAR) AS SELECT * FROM MULTITENANT_TEST_TABLE_WITH_SALT 20 | UPSERT INTO TENANT_VIEW_WITH_SALT (ORGANIZATION_ID, TENANT_ONLY_COL) VALUES ('defaultOrg', 'defaultData') 21 | -------------------------------------------------------------------------------- /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | name: phoenix-connectors preCommit Build 18 | 19 | on: 20 | pull_request: 21 | 22 | jobs: 23 | build: 24 | # upgrade to ubuntu-latest after removing Phoenix 4 support 25 | runs-on: ubuntu-24.04 26 | steps: 27 | - uses: actions/checkout@v1 28 | - name: Set up JDK 1.8 29 | uses: actions/setup-java@v4 30 | with: 31 | java-version: 8 32 | distribution: 'temurin' 33 | - name: Test 34 | run: | 35 | mvn -v 36 | mvn -B -fae clean install 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 17 | 18 | ![logo](https://phoenix.apache.org/images/phoenix-logo-small.png) 19 | 20 | [Apache Phoenix](http://phoenix.apache.org/) enables OLTP and operational analytics in Hadoop for low latency applications. Visit the Apache Phoenix website [here](http://phoenix.apache.org/). This repo contains connectors for third party libraries to access data stored in Phoenix/HBase. 21 | 22 | Copyright ©2019 [Apache Software Foundation](http://www.apache.org/). All Rights Reserved. 23 | 24 | ## Introduction 25 | This repo contains the Spark and Hive connectors for Phoenix. -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDateObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; 21 | 22 | /** 23 | * ObjectInspector for date type 24 | */ 25 | 26 | public class PhoenixDateObjectInspector extends PhoenixDateObjectInspectorBase 27 | implements DateObjectInspector { 28 | 29 | public PhoenixDateObjectInspector() { 30 | super(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDateObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; 21 | 22 | /** 23 | * ObjectInspector for date type 24 | */ 25 | 26 | public class PhoenixDateObjectInspector extends PhoenixDateObjectInspectorBase 27 | implements DateObjectInspector { 28 | 29 | public PhoenixDateObjectInspector() { 30 | super(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixTimestampObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; 21 | 22 | /** 23 | * ObjectInspector for timestamp type 24 | */ 25 | public class PhoenixTimestampObjectInspector extends PhoenixTimestampObjectInspectorBase 26 | implements TimestampObjectInspector { 27 | 28 | public PhoenixTimestampObjectInspector() { super(); } 29 | } 30 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixTimestampObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector; 21 | 22 | /** 23 | * ObjectInspector for timestamp type 24 | */ 25 | public class PhoenixTimestampObjectInspector extends PhoenixTimestampObjectInspectorBase 26 | implements TimestampObjectInspector { 27 | 28 | public PhoenixTimestampObjectInspector() { super(); } 29 | } 30 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/dataset/Dataset.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hive.ql.dataset; 19 | 20 | /** 21 | * Dataset: simple class representation of a dataset 22 | */ 23 | public class Dataset { 24 | public static final String INIT_FILE_NAME = "load.hive.sql"; 25 | public static final String CLEANUP_FILE_NAME = "cleanup.hive.sql"; 26 | 27 | private String table; 28 | 29 | public Dataset(String table) { 30 | this.table = table; 31 | } 32 | 33 | public String getTable(){ 34 | return table; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /phoenix5-hive/src/it/java/org/apache/phoenix/hive/HiveTezIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.phoenix.hive; 20 | 21 | import org.junit.BeforeClass; 22 | import org.junit.Ignore; 23 | 24 | // I was unable to find a combination classpath that works with Phoenix 5.2, and the unrelocated 25 | // hive-exec uberjar that is added by the tez job runner. 26 | @Ignore 27 | public class HiveTezIT extends HivePhoenixStoreIT { 28 | 29 | @BeforeClass 30 | public static void setUpBeforeClass() throws Exception { 31 | setup(HiveTestUtil.MiniClusterType.tez); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/phoenix/hive/HiveTezIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.phoenix.hive; 20 | 21 | import org.apache.hadoop.hive.ql.QTestMiniClusters; 22 | import org.junit.BeforeClass; 23 | import org.junit.Ignore; 24 | 25 | @Ignore 26 | // This time, we cannot run tests because of unshaded protobuf conflict between 27 | // HBase (2.5.0) and Tez (3.x) 28 | public class HiveTezIT extends HivePhoenixStoreIT { 29 | 30 | @BeforeClass 31 | public static void setUpBeforeClass() throws Exception { 32 | setup(QTestMiniClusters.MiniClusterType.TEZ); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/scala/org/apache/phoenix/spark/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | package org.apache.phoenix 15 | 16 | import org.apache.spark.SparkContext 17 | import org.apache.spark.rdd.RDD 18 | import org.apache.spark.sql.{DataFrame, SQLContext} 19 | 20 | package object spark { 21 | implicit def toProductRDDFunctions[A <: Product](rdd: RDD[A]): ProductRDDFunctions[A] = { 22 | new ProductRDDFunctions[A](rdd) 23 | } 24 | 25 | implicit def toSparkContextFunctions(sc: SparkContext): SparkContextFunctions = { 26 | new SparkContextFunctions(sc) 27 | } 28 | 29 | implicit def toSparkSqlContextFunctions(sqlContext: SQLContext): SparkSqlContextFunctions = { 30 | new SparkSqlContextFunctions(sqlContext) 31 | } 32 | 33 | implicit def toDataFrameFunctions(data: DataFrame): DataFrameFunctions = { 34 | new DataFrameFunctions(data) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/scala/org/apache/phoenix/spark/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | package org.apache.phoenix 15 | 16 | import org.apache.spark.SparkContext 17 | import org.apache.spark.rdd.RDD 18 | import org.apache.spark.sql.{DataFrame, SQLContext} 19 | 20 | package object spark { 21 | implicit def toProductRDDFunctions[A <: Product](rdd: RDD[A]): ProductRDDFunctions[A] = { 22 | new ProductRDDFunctions[A](rdd) 23 | } 24 | 25 | implicit def toSparkContextFunctions(sc: SparkContext): SparkContextFunctions = { 26 | new SparkContextFunctions(sc) 27 | } 28 | 29 | implicit def toSparkSqlContextFunctions(sqlContext: SQLContext): SparkSqlContextFunctions = { 30 | new SparkSqlContextFunctions(sqlContext) 31 | } 32 | 33 | implicit def toDataFrameFunctions(data: DataFrame): DataFrameFunctions = { 34 | new DataFrameFunctions(data) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /.asf.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This file controls the integration of the Phoenix Connectors repo 18 | # with ASF infrastructure. Refer to 19 | # https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features 20 | # for details. Be careful when changing the contents of this file since it 21 | # may affect many developers of the project and make sure to discuss the 22 | # changes with dev@ before committing. 23 | 24 | notifications: 25 | commits: commits@phoenix.apache.org 26 | issues: issues@phoenix.apache.org 27 | pullrequests: issues@phoenix.apache.org 28 | jira_options: link label comment 29 | 30 | github: 31 | enabled_merge_buttons: 32 | squash: true 33 | merge: false 34 | rebase: false 35 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/reader/PhoenixTestScanBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.reader; 19 | 20 | import org.apache.spark.sql.connector.read.Scan; 21 | import org.apache.spark.sql.types.StructType; 22 | import org.apache.spark.sql.util.CaseInsensitiveStringMap; 23 | 24 | public class PhoenixTestScanBuilder extends PhoenixScanBuilder { 25 | public PhoenixTestScanBuilder(StructType schema, CaseInsensitiveStringMap options) { 26 | super(schema, options); 27 | } 28 | 29 | @Override 30 | public Scan build() { 31 | return new PhoenixTestScan(getSchema(), getOptions(), getWhereClause()); 32 | } 33 | } -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/writer/PhoenixTestWriteBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.writer; 19 | 20 | import org.apache.spark.sql.connector.write.BatchWrite; 21 | import org.apache.spark.sql.connector.write.LogicalWriteInfo; 22 | 23 | import java.util.Map; 24 | 25 | public class PhoenixTestWriteBuilder extends PhoenixWriteBuilder { 26 | public PhoenixTestWriteBuilder(LogicalWriteInfo info, Map options) { 27 | super(info, options); 28 | } 29 | 30 | @Override 31 | public BatchWrite buildForBatch() { 32 | return new PhoenixTestBatchWrite(getWriteInfo(), getOptions()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/java/org/apache/phoenix/spark/datasource/v2/writer/PhoenixTestingDataWriterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.writer; 19 | 20 | import org.apache.spark.sql.catalyst.InternalRow; 21 | import org.apache.spark.sql.sources.v2.writer.DataWriter; 22 | 23 | public class PhoenixTestingDataWriterFactory extends PhoenixDataWriterFactory { 24 | 25 | PhoenixTestingDataWriterFactory(PhoenixDataSourceWriteOptions options) { 26 | super(options); 27 | } 28 | 29 | // Override to return a test DataWriter 30 | @Override 31 | public DataWriter createDataWriter(int partitionId, long taskId, long epochId) { 32 | return new PhoenixTestingDataWriter(getOptions()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/java/org/apache/phoenix/spark/sql/connector/reader/PhoenixInputPartition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.reader; 19 | 20 | import org.apache.phoenix.mapreduce.PhoenixInputSplit; 21 | import org.apache.spark.SerializableWritable; 22 | import org.apache.spark.sql.connector.read.InputPartition; 23 | 24 | public class PhoenixInputPartition implements InputPartition { 25 | 26 | private final SerializableWritable phoenixInputSplit; 27 | 28 | PhoenixInputPartition(PhoenixInputSplit phoenixInputSplit) { 29 | this.phoenixInputSplit = new SerializableWritable<>(phoenixInputSplit); 30 | } 31 | 32 | SerializableWritable getPhoenixInputSplit() { 33 | return phoenixInputSplit; 34 | } 35 | } -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/writer/PhoenixTestDataWriterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.writer; 19 | 20 | import org.apache.spark.sql.catalyst.InternalRow; 21 | import org.apache.spark.sql.connector.write.DataWriter; 22 | import org.apache.spark.sql.types.StructType; 23 | 24 | public class PhoenixTestDataWriterFactory extends PhoenixDataWriterFactory { 25 | PhoenixTestDataWriterFactory(StructType schema, PhoenixDataSourceWriteOptions options) { 26 | super(schema, options); 27 | } 28 | 29 | // Override to return a test DataWriter 30 | @Override 31 | public DataWriter createWriter(int partitionId, long taskId) { 32 | return new PhoenixTestDataWriter(getSchema(), getOptions()); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /phoenix5-hive/src/it/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Define the root logger to the system property "hbase.root.logger". 18 | log4j.rootLogger=INFO,console 19 | 20 | # Logging Threshold 21 | log4j.threshhold=ALL 22 | 23 | # 24 | # console 25 | # Add "console" to rootlogger above if you want to use this 26 | # 27 | log4j.appender.console=org.apache.log4j.ConsoleAppender 28 | log4j.appender.console.target=System.err 29 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 30 | log4j.appender.console.layout.ConversionPattern=%d %-5p %C(%L): %m%n 31 | 32 | # Custom Logging levels 33 | 34 | log4j.logger.org.apache.zookeeper=ERROR 35 | log4j.logger.org.apache.hadoop.hbase=DEBUG 36 | log4j.logger.org.apache.phoenix.hive=DEBUG 37 | log4j.logger.org.apache.phoenix.hive.query=DEBUG 38 | log4j.logger.org.apache.phoenix.hive.objectinspector=DEBUG 39 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Define the root logger to the system property "hbase.root.logger". 18 | log4j.rootLogger=INFO,console 19 | 20 | # Logging Threshold 21 | log4j.threshhold=ALL 22 | 23 | # 24 | # console 25 | # Add "console" to rootlogger above if you want to use this 26 | # 27 | log4j.appender.console=org.apache.log4j.ConsoleAppender 28 | log4j.appender.console.target=System.err 29 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 30 | log4j.appender.console.layout.ConversionPattern=%d %-5p %C(%L): %m%n 31 | 32 | # Custom Logging levels 33 | 34 | log4j.logger.org.apache.zookeeper=ERROR 35 | log4j.logger.org.apache.hadoop.hbase=DEBUG 36 | log4j.logger.org.apache.phoenix.hive=DEBUG 37 | log4j.logger.org.apache.phoenix.hive.query=DEBUG 38 | log4j.logger.org.apache.phoenix.hive.objectinspector=DEBUG 39 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/java/org/apache/phoenix/spark/datasource/v2/reader/PhoenixTestingDataSourceReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.reader; 19 | 20 | import org.apache.phoenix.mapreduce.PhoenixInputSplit; 21 | import org.apache.spark.sql.sources.v2.DataSourceOptions; 22 | 23 | public class PhoenixTestingDataSourceReader extends PhoenixDataSourceReader { 24 | 25 | public PhoenixTestingDataSourceReader(DataSourceOptions options) { 26 | super(options); 27 | } 28 | 29 | // Override to return a test InputPartition 30 | @Override 31 | PhoenixInputPartition getInputPartition(PhoenixDataSourceReadOptions readOptions, 32 | PhoenixInputSplit inputSplit) { 33 | return new PhoenixTestingInputPartition(readOptions, readSchema(), inputSplit); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/phoenix/hive/HiveMapReduceIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.phoenix.hive; 20 | 21 | import static org.junit.Assert.fail; 22 | 23 | import org.apache.hadoop.hive.ql.QTestMiniClusters; 24 | import org.junit.BeforeClass; 25 | import org.junit.Ignore; 26 | import org.junit.Test; 27 | 28 | public class HiveMapReduceIT extends HivePhoenixStoreIT { 29 | 30 | @BeforeClass 31 | public static void setUpBeforeClass() throws Exception { 32 | setup(QTestMiniClusters.MiniClusterType.MR); 33 | } 34 | 35 | @Override 36 | @Test 37 | @Ignore 38 | /** 39 | * Ignoring because projection pushdown is incorrect for MR when there are multiple aliases (ref:HIVE-18872) 40 | */ 41 | public void testJoinColumnMaps() throws Exception { 42 | 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /phoenix5-connectors-assembly/src/build/package-phoenix5-connectors-to-tar-unshaded.xml: -------------------------------------------------------------------------------- 1 | 2 | 22 | 23 | 26 | 27 | unshaded-only 28 | 29 | tar.gz 30 | 31 | true 32 | 33 | 34 | 35 | src/build/components/phoenix5-jars.xml 36 | 37 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/reader/PhoenixTestScan.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.reader; 19 | 20 | import org.apache.spark.sql.connector.read.Batch; 21 | import org.apache.spark.sql.connector.read.PartitionReaderFactory; 22 | import org.apache.spark.sql.types.StructType; 23 | import org.apache.spark.sql.util.CaseInsensitiveStringMap; 24 | 25 | public class PhoenixTestScan extends PhoenixScan { 26 | PhoenixTestScan(StructType schema, CaseInsensitiveStringMap options, String whereClause) { 27 | super(schema, options, whereClause); 28 | } 29 | 30 | @Override 31 | public Batch toBatch() { 32 | return this; 33 | } 34 | 35 | @Override 36 | public PartitionReaderFactory createReaderFactory() { 37 | return new PhoenixTestPartitionReadFactory(getOptions(), readSchema()); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/java/org/apache/phoenix/spark/datasource/v2/writer/PhoenixDataWriterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.writer; 19 | 20 | import org.apache.spark.sql.catalyst.InternalRow; 21 | import org.apache.spark.sql.sources.v2.writer.DataWriter; 22 | import org.apache.spark.sql.sources.v2.writer.DataWriterFactory; 23 | 24 | public class PhoenixDataWriterFactory implements DataWriterFactory { 25 | 26 | private final PhoenixDataSourceWriteOptions options; 27 | 28 | PhoenixDataWriterFactory(PhoenixDataSourceWriteOptions options) { 29 | this.options = options; 30 | } 31 | 32 | PhoenixDataSourceWriteOptions getOptions() { 33 | return options; 34 | } 35 | 36 | @Override 37 | public DataWriter createDataWriter(int partitionId, long taskId, long epochId) { 38 | return new PhoenixDataWriter(options); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/writer/PhoenixTestingWriterCommitMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.writer; 19 | 20 | import org.apache.spark.sql.connector.write.WriterCommitMessage; 21 | 22 | public class PhoenixTestingWriterCommitMessage implements WriterCommitMessage { 23 | private long numBatchesCommitted = 0; 24 | PhoenixTestingWriterCommitMessage(long numBatchesCommitted) { 25 | this.numBatchesCommitted = numBatchesCommitted; 26 | } 27 | 28 | // Override to keep track of the number of batches committed by the corresponding DataWriter 29 | // in the WriterCommitMessage, so we can observe this value in the driver when we call 30 | // {@link PhoenixTestingDataSourceWriter#commit(WriterCommitMessage[])} 31 | @Override 32 | public String toString() { 33 | return String.valueOf(this.numBatchesCommitted); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/java/org/apache/phoenix/spark/datasource/v2/writer/PhoenixTestingWriterCommitMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.writer; 19 | 20 | import org.apache.spark.sql.sources.v2.writer.WriterCommitMessage; 21 | 22 | class PhoenixTestingWriterCommitMessage implements WriterCommitMessage { 23 | 24 | private final long numBatchesCommitted; 25 | 26 | PhoenixTestingWriterCommitMessage(long numBatchesCommitted) { 27 | this.numBatchesCommitted = numBatchesCommitted; 28 | } 29 | 30 | // Override to keep track of the number of batches committed by the corresponding DataWriter 31 | // in the WriterCommitMessage, so we can observe this value in the driver when we call 32 | // {@link PhoenixTestingDataSourceWriter#commit(WriterCommitMessage[])} 33 | @Override 34 | public String toString() { 35 | return String.valueOf(this.numBatchesCommitted); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/ql/pushdown/PredicateAnalyzerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.ql.pushdown; 19 | 20 | import java.util.List; 21 | 22 | import org.apache.phoenix.hive.ql.pushdown.PhoenixPredicateAnalyzer.FieldValidator; 23 | 24 | public class PredicateAnalyzerFactory { 25 | public static PhoenixPredicateAnalyzer createPredicateAnalyzer(List ppdColumnList, 26 | FieldValidator fieldValdator) { 27 | // Create analyzer for conditions =, <, <=, >, >= 28 | PhoenixPredicateAnalyzer analyzer = PhoenixPredicateAnalyzer.createAnalyzer(false); 29 | 30 | for (String columnName : ppdColumnList) { 31 | analyzer.allowColumnName(columnName); 32 | } 33 | 34 | analyzer.setAcceptsFields(true); 35 | analyzer.setFieldValidator(fieldValdator); 36 | 37 | return analyzer; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/ql/pushdown/PredicateAnalyzerFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.ql.pushdown; 19 | 20 | import java.util.List; 21 | 22 | import org.apache.phoenix.hive.ql.pushdown.PhoenixPredicateAnalyzer.FieldValidator; 23 | 24 | public class PredicateAnalyzerFactory { 25 | public static PhoenixPredicateAnalyzer createPredicateAnalyzer(List ppdColumnList, 26 | FieldValidator fieldValdator) { 27 | // Create analyzer for conditions =, <, <=, >, >= 28 | PhoenixPredicateAnalyzer analyzer = PhoenixPredicateAnalyzer.createAnalyzer(false); 29 | 30 | for (String columnName : ppdColumnList) { 31 | analyzer.allowColumnName(columnName); 32 | } 33 | 34 | analyzer.setAcceptsFields(true); 35 | analyzer.setFieldValidator(fieldValdator); 36 | 37 | return analyzer; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /phoenix5-connectors-assembly/src/build/package-phoenix5-connectors-to-tar-all.xml: -------------------------------------------------------------------------------- 1 | 2 | 22 | 23 | 26 | 27 | all 28 | 29 | tar.gz 30 | 31 | true 32 | 33 | 34 | 35 | src/build/components/phoenix5-jars.xml 36 | src/build/components/phoenix5-shaded-jars.xml 37 | 38 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/reader/PhoenixTestPartitionReadFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.reader; 19 | 20 | import org.apache.spark.sql.catalyst.InternalRow; 21 | import org.apache.spark.sql.connector.read.InputPartition; 22 | import org.apache.spark.sql.connector.read.PartitionReader; 23 | import org.apache.spark.sql.types.StructType; 24 | 25 | public class PhoenixTestPartitionReadFactory extends PhoenixPartitionReadFactory { 26 | 27 | PhoenixTestPartitionReadFactory(PhoenixDataSourceReadOptions options, StructType structType) { 28 | super(options, structType); 29 | } 30 | 31 | // Override to return a test InputPartitionReader for testing on the executor-side 32 | @Override 33 | public PartitionReader createReader(InputPartition partition) { 34 | return new PhoenixTestPartitionReader(getOptions(), readSchema(), (PhoenixInputPartition) partition); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/qoption/QTestOptionHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hive.ql.qoption; 19 | 20 | import org.apache.hadoop.hive.ql.QTestUtil; 21 | 22 | /** 23 | * Qtest options might be usefull to prepare the test environment or do some extra checks/cleanup. 24 | */ 25 | public interface QTestOptionHandler { 26 | 27 | /** 28 | * For a matching option; the arguments are supplied to the handler by this method. 29 | */ 30 | void processArguments(String arguments); 31 | 32 | /** 33 | * Invoked before the actual test is executed. 34 | * 35 | * At the time of this call all the options for the actual test is already processed. 36 | */ 37 | void beforeTest(QTestUtil qt) throws Exception; 38 | 39 | /** 40 | * Invoked right after the test is executed. 41 | * 42 | * Can be used to cleanup things and/or clear internal state of the handler. 43 | */ 44 | void afterTest(QTestUtil qt) throws Exception; 45 | 46 | } 47 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/dataset/DatasetCollection.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.hive.ql.dataset; 20 | 21 | import java.util.HashSet; 22 | import java.util.Set; 23 | import java.util.stream.Collectors; 24 | 25 | /** 26 | * DatasetCollection: utility wrapper class for a set of datasets 27 | */ 28 | public class DatasetCollection { 29 | private Set coll = new HashSet(); 30 | 31 | public DatasetCollection() { 32 | } 33 | 34 | public DatasetCollection(Set datasets) { 35 | for (String datasetName : datasets) { 36 | add(datasetName); 37 | } 38 | } 39 | 40 | public void add(Dataset dataset) { 41 | coll.add(dataset); 42 | } 43 | 44 | public void add(String table) { 45 | add(new Dataset(table)); 46 | } 47 | 48 | public Set getDatasets() { 49 | return coll; 50 | } 51 | 52 | public Set getTables() { 53 | return coll.stream().map(d -> d.getTable()).collect(Collectors.toSet()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /dev/docker/Dockerfile.yetus: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | FROM apache/yetus:0.13.0 18 | 19 | ##### 20 | # Update SpotBugs 21 | ##### 22 | RUN rm -rf /opt/spotbugs \ 23 | && git clone https://github.com/stoty/spotbugs.git \ 24 | && cd spotbugs \ 25 | && git checkout PHOENIX-1161-backport \ 26 | && ./gradlew clean build -x test \ 27 | && mkdir /opt/spotbugs \ 28 | && tar -C /opt/spotbugs --strip-components 1 -xpf spotbugs/build/distributions/spotbugs-4.1.2.tgz \ 29 | && chmod a+rx /opt/spotbugs/bin/* \ 30 | && cd .. \ 31 | && rm -rf ./spotbugs 32 | ENV SPOTBUGS_HOME /opt/spotbugs 33 | ##### 34 | # Enable spotbugs prefix for maven 35 | ##### 36 | RUN sed -i -e 's/<\/pluginGroups>/com.github.spotbugs<\/pluginGroup><\/pluginGroups>/' /etc/maven/settings.xml 37 | ##### 38 | RUN mkdir -p /run/user/910/gnupg/ \ 39 | && chmod -R 700 /run/user/910 \ 40 | && chown -R 910:910 /run/user/910 \ 41 | && mkdir -p /var/run/user/910/gnupg \ 42 | && chown -R 910:910 /var/run/user/910 \ 43 | && chmod -R 700 /var/run/user/910 44 | ##### 45 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/reader/PhoenixTestPartitionReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.reader; 19 | 20 | import org.apache.spark.sql.catalyst.InternalRow; 21 | import org.apache.spark.sql.types.StructType; 22 | 23 | import java.util.Properties; 24 | 25 | public class PhoenixTestPartitionReader extends PhoenixPartitionReader { 26 | 27 | public static final String RETURN_NULL_CURR_ROW = "return.null.curr.row"; 28 | 29 | PhoenixTestPartitionReader(PhoenixDataSourceReadOptions options, StructType schema , PhoenixInputPartition inputPartition) { 30 | super(options, schema, inputPartition); 31 | } 32 | 33 | // Override to return null rather than the actual row based on a property passed to the executor 34 | @Override 35 | public InternalRow get() { 36 | Properties props = getOverriddenPropsFromOptions(); 37 | return Boolean.parseBoolean(props.getProperty(RETURN_NULL_CURR_ROW)) ? null : super.get(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/PhoenixJdbcDialect.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.spark.sql.execution.datasources.jdbc 20 | 21 | import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcType} 22 | import org.apache.spark.sql.types.{BinaryType, ByteType, DataType, StringType, ShortType} 23 | 24 | private object PhoenixJdbcDialect extends JdbcDialect { 25 | 26 | override def canHandle(url: String): Boolean = url.startsWith("jdbc:phoenix") 27 | 28 | /** 29 | * This is only called for ArrayType (see JdbcUtils.makeSetter) 30 | */ 31 | override def getJDBCType(dt: DataType): Option[JdbcType] = dt match { 32 | case StringType => Some(JdbcType("VARCHAR", java.sql.Types.VARCHAR)) 33 | case BinaryType => Some(JdbcType("BINARY(" + dt.defaultSize + ")", java.sql.Types.BINARY)) 34 | case ByteType => Some(JdbcType("TINYINT", java.sql.Types.TINYINT)) 35 | case ShortType => Some(JdbcType("SMALLINT", java.sql.Types.SMALLINT)) 36 | case _ => None 37 | } 38 | 39 | 40 | } 41 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/PhoenixJdbcDialect.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.spark.sql.execution.datasources.jdbc 20 | 21 | import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcType} 22 | import org.apache.spark.sql.types.{BinaryType, ByteType, DataType, StringType, ShortType} 23 | 24 | private object PhoenixJdbcDialect extends JdbcDialect { 25 | 26 | override def canHandle(url: String): Boolean = url.startsWith("jdbc:phoenix") 27 | 28 | /** 29 | * This is only called for ArrayType (see JdbcUtils.makeSetter) 30 | */ 31 | override def getJDBCType(dt: DataType): Option[JdbcType] = dt match { 32 | case StringType => Some(JdbcType("VARCHAR", java.sql.Types.VARCHAR)) 33 | case BinaryType => Some(JdbcType("BINARY(" + dt.defaultSize + ")", java.sql.Types.BINARY)) 34 | case ByteType => Some(JdbcType("TINYINT", java.sql.Types.TINYINT)) 35 | case ShortType => Some(JdbcType("SMALLINT", java.sql.Types.SMALLINT)) 36 | case _ => None 37 | } 38 | 39 | 40 | } 41 | -------------------------------------------------------------------------------- /phoenix5-hive/src/it/java/org/apache/phoenix/hive/HiveMapReduceIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.phoenix.hive; 20 | 21 | import static org.junit.Assert.fail; 22 | 23 | import org.junit.BeforeClass; 24 | import org.junit.Ignore; 25 | import org.junit.Test; 26 | 27 | public class HiveMapReduceIT extends HivePhoenixStoreIT { 28 | 29 | @BeforeClass 30 | public static void setUpBeforeClass() throws Exception { 31 | setup(HiveTestUtil.MiniClusterType.mr); 32 | } 33 | 34 | @Override 35 | @Test 36 | @Ignore 37 | /** 38 | * Ignoring because precicate pushdown is skipped for MR (ref:HIVE-18873) when there are multiple aliases 39 | */ 40 | public void testJoinNoColumnMaps() throws Exception { 41 | 42 | } 43 | 44 | @Override 45 | @Test 46 | @Ignore 47 | /** 48 | * Ignoring because projection pushdown is incorrect for MR when there are multiple aliases (ref:HIVE-18872) 49 | */ 50 | public void testJoinColumnMaps() throws Exception { 51 | 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/qoption/QTestDisabledHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hive.ql.qoption; 19 | 20 | import org.apache.hadoop.hive.ql.QTestUtil; 21 | import org.junit.Assume; 22 | 23 | import com.google.common.base.Strings; 24 | 25 | /** 26 | * QTest disabled directive handler 27 | * 28 | * Example: 29 | * --! qt:disabled:reason 30 | * 31 | */ 32 | public class QTestDisabledHandler implements QTestOptionHandler { 33 | 34 | private String message; 35 | 36 | @Override 37 | public void processArguments(String arguments) { 38 | message = arguments; 39 | if (Strings.isNullOrEmpty(message)) { 40 | throw new RuntimeException("you have to give a reason why it was ignored"); 41 | } 42 | } 43 | 44 | @Override 45 | public void beforeTest(QTestUtil qt) throws Exception { 46 | Assume.assumeTrue(message, (message == null)); 47 | } 48 | 49 | @Override 50 | public void afterTest(QTestUtil qt) throws Exception { 51 | message = null; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/java/org/apache/phoenix/spark/datasource/v2/reader/PhoenixTestingInputPartition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.reader; 19 | 20 | import org.apache.phoenix.mapreduce.PhoenixInputSplit; 21 | import org.apache.spark.sql.catalyst.InternalRow; 22 | import org.apache.spark.sql.sources.v2.reader.InputPartitionReader; 23 | import org.apache.spark.sql.types.StructType; 24 | 25 | public class PhoenixTestingInputPartition extends PhoenixInputPartition { 26 | 27 | PhoenixTestingInputPartition(PhoenixDataSourceReadOptions options, StructType schema, 28 | PhoenixInputSplit phoenixInputSplit) { 29 | super(options, schema, phoenixInputSplit); 30 | } 31 | 32 | // Override to return a test InputPartitionReader for testing on the executor-side 33 | @Override 34 | public InputPartitionReader createPartitionReader() { 35 | return new PhoenixTestingInputPartitionReader(getOptions(), getSchema(), 36 | getPhoenixInputSplit()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /dev/rebuild_hbase.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | 14 | # Rebuilds HBase with -Dhadoop.profile=3.0 locally, to work around PHOENIX-5993 15 | # Intended mainly for CI jobs, but can simplify manual rebuilds as well. 16 | 17 | # The name of the Apache Hbase source file 18 | HBASE_SOURCE_NAME="hbase-$1-src.tar.gz" 19 | # The relative path on the ASF mirrors for the Hbase source file 20 | HBASE_SOURCE_MIRROR_NAME="hbase/$1/$HBASE_SOURCE_NAME" 21 | 22 | # Downloads the specified HBase version source, extracts it, 23 | # then rebuilds and installs the maven artifacts locally with -Dhadoop.profile=3.0 24 | 25 | if [ $# -ne 1 ] 26 | then 27 | echo "Supply the Hbase version as paramater i.e.: rebuild_hbase.sh 2.2.6 " 28 | fi 29 | 30 | DEV_SUPPORT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 31 | ARTIFACTS_DIR="$DEV_SUPPORT/artifacts" 32 | WORK_DIR="$DEV_SUPPORT/work" 33 | 34 | mkdir "$ARTIFACTS_DIR" 35 | mkdir "$WORK_DIR" 36 | 37 | $DEV_SUPPORT/cache-apache-project-artifact.sh --keys https://downloads.apache.org/hbase/KEYS \ 38 | --working-dir "$WORK_DIR" "$ARTIFACTS_DIR/$HBASE_SOURCE_NAME" "$HBASE_SOURCE_MIRROR_NAME" 39 | 40 | STARTDIR=$PWD 41 | cd $ARTIFACTS_DIR 42 | tar xfz hbase-$1-src.tar.gz 43 | cd hbase-$1 44 | mvn clean install -Dhadoop.profile=3.0 -DskipTests -B 45 | cd ${STARTDIR} 46 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/java/org/apache/phoenix/spark/datasource/v2/writer/PhoenixTestingDataWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.writer; 19 | 20 | import org.apache.spark.sql.sources.v2.writer.WriterCommitMessage; 21 | 22 | import java.sql.SQLException; 23 | 24 | public class PhoenixTestingDataWriter extends PhoenixDataWriter { 25 | 26 | private long numBatchesCommitted = 0; 27 | 28 | PhoenixTestingDataWriter(PhoenixDataSourceWriteOptions options) { 29 | super(options); 30 | } 31 | 32 | // Override to also count the number of times we call this method to test upsert batch commits 33 | @Override 34 | void commitBatchUpdates() throws SQLException { 35 | super.commitBatchUpdates(); 36 | numBatchesCommitted++; 37 | } 38 | 39 | // Override to return a test WriterCommitMessage 40 | @Override 41 | public WriterCommitMessage commit() { 42 | super.commit(); 43 | return new PhoenixTestingWriterCommitMessage(numBatchesCommitted); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/writer/PhoenixTestDataWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.writer; 19 | 20 | import org.apache.spark.sql.connector.write.WriterCommitMessage; 21 | import org.apache.spark.sql.types.StructType; 22 | 23 | import java.sql.SQLException; 24 | 25 | public class PhoenixTestDataWriter extends PhoenixDataWriter { 26 | private long numBatchesCommitted = 0; 27 | PhoenixTestDataWriter(StructType schema, PhoenixDataSourceWriteOptions options) { 28 | super(schema, options); 29 | } 30 | 31 | // Override to also count the number of times we call this method to test upsert batch commits 32 | @Override 33 | public void commitBatchUpdates() throws SQLException { 34 | super.commitBatchUpdates(); 35 | numBatchesCommitted++; 36 | } 37 | 38 | // Override to return a test WriterCommitMessage 39 | @Override 40 | public WriterCommitMessage commit() { 41 | super.commit(); 42 | return new PhoenixTestingWriterCommitMessage(numBatchesCommitted); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /phoenix5-hive/src/it/java/org/apache/phoenix/hive/HiveTestUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive; 19 | 20 | import org.apache.hadoop.hive.ql.QTestUtil; 21 | 22 | /** 23 | * HiveTestUtil cloned from Hive QTestUtil. Can be outdated and may require update once a problem 24 | * found. 25 | */ 26 | public class HiveTestUtil extends QTestUtil{ 27 | public static final String CRLF = System.getProperty("line.separator"); 28 | 29 | public HiveTestUtil(String outDir, String logDir, QTestUtil.MiniClusterType clusterType, String confDir, String hadoopVer, 30 | String initScript, String cleanupScript, boolean withLlapIo) throws Exception { 31 | super(outDir, logDir, clusterType, confDir, hadoopVer, initScript, cleanupScript, withLlapIo); 32 | } 33 | 34 | @Override 35 | public int executeClient(String tname) { 36 | conf.set("mapreduce.job.name", "test"); 37 | conf.set("hive.mapred.mode", "nonstrict"); 38 | conf.set("hive.strict.checks.cartesian.product", "false"); 39 | return super.executeClient(tname); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/PhoenixTestingTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector; 19 | 20 | import org.apache.phoenix.spark.sql.connector.reader.PhoenixTestScanBuilder; 21 | import org.apache.phoenix.spark.sql.connector.writer.PhoenixTestWriteBuilder; 22 | import org.apache.spark.sql.connector.read.ScanBuilder; 23 | import org.apache.spark.sql.connector.write.LogicalWriteInfo; 24 | import org.apache.spark.sql.connector.write.WriteBuilder; 25 | import org.apache.spark.sql.types.StructType; 26 | import org.apache.spark.sql.util.CaseInsensitiveStringMap; 27 | 28 | import java.util.Map; 29 | 30 | 31 | public class PhoenixTestingTable extends PhoenixTable { 32 | 33 | PhoenixTestingTable(StructType schema, Map options) { 34 | super(schema, options); 35 | } 36 | 37 | @Override 38 | public ScanBuilder newScanBuilder(CaseInsensitiveStringMap options) { 39 | return new PhoenixTestScanBuilder(schema(), options); 40 | } 41 | 42 | @Override 43 | public WriteBuilder newWriteBuilder(LogicalWriteInfo info) { 44 | return new PhoenixTestWriteBuilder(info,getOptions()); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/qoption/QTestTransactional.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.hive.ql.qoption; 20 | 21 | import org.apache.hadoop.hive.ql.QTestUtil; 22 | import org.slf4j.Logger; 23 | import org.slf4j.LoggerFactory; 24 | 25 | /** 26 | * QTest transactional directive handler 27 | * 28 | * Enables transactional for the test. 29 | * Could also make it for other QOption-s. 30 | * 31 | * Example: 32 | * --! qt:transactional 33 | * 34 | */ 35 | public class QTestTransactional implements QTestOptionHandler { 36 | private static final Logger LOG = LoggerFactory.getLogger(QTestTransactional.class.getName()); 37 | private boolean enabled; 38 | 39 | @Override 40 | public void processArguments(String arguments) { 41 | enabled = true; 42 | } 43 | 44 | @Override 45 | public void beforeTest(QTestUtil qt) throws Exception { 46 | if (enabled) { 47 | qt.getConf().set("hive.support.concurrency", "true"); 48 | qt.getConf().set("hive.txn.manager", "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager"); 49 | } 50 | } 51 | 52 | @Override 53 | public void afterTest(QTestUtil qt) throws Exception { 54 | enabled = false; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/java/org/apache/phoenix/spark/sql/connector/writer/PhoenixDataWriterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.writer; 19 | 20 | import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting; 21 | import org.apache.spark.sql.catalyst.InternalRow; 22 | import org.apache.spark.sql.connector.write.DataWriter; 23 | import org.apache.spark.sql.connector.write.DataWriterFactory; 24 | import org.apache.spark.sql.types.StructType; 25 | 26 | public class PhoenixDataWriterFactory implements DataWriterFactory { 27 | 28 | private final StructType schema; 29 | private final PhoenixDataSourceWriteOptions options; 30 | 31 | PhoenixDataWriterFactory(StructType schema, PhoenixDataSourceWriteOptions options) { 32 | this.schema = schema; 33 | this.options = options; 34 | } 35 | 36 | @Override 37 | public DataWriter createWriter(int partitionId, long taskId) { 38 | return new PhoenixDataWriter(schema, options); 39 | } 40 | 41 | @VisibleForTesting 42 | PhoenixDataSourceWriteOptions getOptions(){ 43 | return options; 44 | } 45 | 46 | @VisibleForTesting 47 | StructType getSchema() { 48 | return schema; 49 | } 50 | } -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.LongWritable; 23 | 24 | public class PhoenixLongObjectInspector extends AbstractPhoenixObjectInspector 25 | implements LongObjectInspector { 26 | 27 | public PhoenixLongObjectInspector() { 28 | super(TypeInfoFactory.longTypeInfo); 29 | } 30 | 31 | @Override 32 | public Object copyObject(Object o) { 33 | return o == null ? null : new Long((Long) o); 34 | } 35 | 36 | @Override 37 | public LongWritable getPrimitiveWritableObject(Object o) { 38 | return new LongWritable(get(o)); 39 | } 40 | 41 | @Override 42 | public long get(Object o) { 43 | Long value = null; 44 | 45 | if (o != null) { 46 | try { 47 | value = ((Long) o).longValue(); 48 | } catch (Exception e) { 49 | logExceptionMessage(o, "LONG"); 50 | } 51 | } 52 | 53 | return value; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixLongObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.LongWritable; 23 | 24 | public class PhoenixLongObjectInspector extends AbstractPhoenixObjectInspector 25 | implements LongObjectInspector { 26 | 27 | public PhoenixLongObjectInspector() { 28 | super(TypeInfoFactory.longTypeInfo); 29 | } 30 | 31 | @Override 32 | public Object copyObject(Object o) { 33 | return o == null ? null : new Long((Long) o); 34 | } 35 | 36 | @Override 37 | public LongWritable getPrimitiveWritableObject(Object o) { 38 | return new LongWritable(get(o)); 39 | } 40 | 41 | @Override 42 | public long get(Object o) { 43 | Long value = null; 44 | 45 | if (o != null) { 46 | try { 47 | value = ((Long) o).longValue(); 48 | } catch (Exception e) { 49 | logExceptionMessage(o, "LONG"); 50 | } 51 | } 52 | 53 | return value; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/scala/org/apache/phoenix/spark/SparkSqlContextFunctions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | package org.apache.phoenix.spark 15 | 16 | import org.apache.hadoop.conf.Configuration 17 | import org.apache.spark.sql.{DataFrame, SQLContext} 18 | 19 | @deprecated("Use the DataSource V2 API implementation (see PhoenixDataSource)") 20 | class SparkSqlContextFunctions(@transient val sqlContext: SQLContext) extends Serializable { 21 | /* 22 | This will return a Spark DataFrame, with Phoenix types converted Spark SQL catalyst types 23 | 24 | 'table' is the corresponding Phoenix table 25 | 'columns' is a sequence of of columns to query 26 | 'predicate' is a set of statements to go after a WHERE clause, e.g. "TID = 123" 27 | 'zkUrl' is an optional Zookeeper URL to use to connect to Phoenix 28 | 'conf' is a Hadoop Configuration object. If zkUrl is not set, the "hbase.zookeeper.quorum" 29 | property will be used 30 | */ 31 | def phoenixTableAsDataFrame(table: String, columns: Seq[String], 32 | predicate: Option[String] = None, 33 | zkUrl: Option[String] = None, 34 | tenantId: Option[String] = None, 35 | conf: Configuration = new Configuration): DataFrame = { 36 | val df = PhoenixDataFrameHelper.createDataFrame(table, zkUrl, tenantId, conf)(sqlContext.sparkSession) 37 | val dfWithSelectColumns = PhoenixDataFrameHelper.withSelectExpr(columns, df) 38 | PhoenixDataFrameHelper.withWhereCondition(predicate, dfWithSelectColumns) 39 | } 40 | 41 | } -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.BooleanWritable; 23 | 24 | public class PhoenixBooleanObjectInspector extends AbstractPhoenixObjectInspector 25 | implements BooleanObjectInspector { 26 | 27 | public PhoenixBooleanObjectInspector() { 28 | super(TypeInfoFactory.booleanTypeInfo); 29 | } 30 | 31 | @Override 32 | public Object copyObject(Object o) { 33 | return o == null ? null : new Boolean((Boolean) o); 34 | } 35 | 36 | @Override 37 | public BooleanWritable getPrimitiveWritableObject(Object o) { 38 | return new BooleanWritable(get(o)); 39 | } 40 | 41 | @Override 42 | public boolean get(Object o) { 43 | Boolean value = null; 44 | 45 | if (o != null) { 46 | try { 47 | value = (Boolean) o; 48 | } catch (Exception e) { 49 | logExceptionMessage(o, "BOOLEAN"); 50 | } 51 | } 52 | 53 | return value; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.io.ShortWritable; 21 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; 22 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 23 | 24 | public class PhoenixShortObjectInspector extends AbstractPhoenixObjectInspector 25 | implements ShortObjectInspector { 26 | 27 | public PhoenixShortObjectInspector() { 28 | super(TypeInfoFactory.shortTypeInfo); 29 | } 30 | 31 | @Override 32 | public Object copyObject(Object o) { 33 | return o == null ? null : new Short((Short) o); 34 | } 35 | 36 | @Override 37 | public ShortWritable getPrimitiveWritableObject(Object o) { 38 | return new ShortWritable(get(o)); 39 | } 40 | 41 | @Override 42 | public short get(Object o) { 43 | Short value = null; 44 | 45 | if (o != null) { 46 | try { 47 | value = ((Short) o).shortValue(); 48 | } catch (Exception e) { 49 | logExceptionMessage(o, "SHORT"); 50 | } 51 | } 52 | 53 | return value; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBooleanObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.BooleanWritable; 23 | 24 | public class PhoenixBooleanObjectInspector extends AbstractPhoenixObjectInspector 25 | implements BooleanObjectInspector { 26 | 27 | public PhoenixBooleanObjectInspector() { 28 | super(TypeInfoFactory.booleanTypeInfo); 29 | } 30 | 31 | @Override 32 | public Object copyObject(Object o) { 33 | return o == null ? null : new Boolean((Boolean) o); 34 | } 35 | 36 | @Override 37 | public BooleanWritable getPrimitiveWritableObject(Object o) { 38 | return new BooleanWritable(get(o)); 39 | } 40 | 41 | @Override 42 | public boolean get(Object o) { 43 | Boolean value = null; 44 | 45 | if (o != null) { 46 | try { 47 | value = (Boolean) o; 48 | } catch (Exception e) { 49 | logExceptionMessage(o, "BOOLEAN"); 50 | } 51 | } 52 | 53 | return value; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixShortObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.io.ShortWritable; 21 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector; 22 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 23 | 24 | public class PhoenixShortObjectInspector extends AbstractPhoenixObjectInspector 25 | implements ShortObjectInspector { 26 | 27 | public PhoenixShortObjectInspector() { 28 | super(TypeInfoFactory.shortTypeInfo); 29 | } 30 | 31 | @Override 32 | public Object copyObject(Object o) { 33 | return o == null ? null : new Short((Short) o); 34 | } 35 | 36 | @Override 37 | public ShortWritable getPrimitiveWritableObject(Object o) { 38 | return new ShortWritable(get(o)); 39 | } 40 | 41 | @Override 42 | public short get(Object o) { 43 | Short value = null; 44 | 45 | if (o != null) { 46 | try { 47 | value = ((Short) o).shortValue(); 48 | } catch (Exception e) { 49 | logExceptionMessage(o, "SHORT"); 50 | } 51 | } 52 | 53 | return value; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBinaryObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.BytesWritable; 23 | 24 | /** 25 | * ObjectInspector for Binary type 26 | */ 27 | 28 | public class PhoenixBinaryObjectInspector extends AbstractPhoenixObjectInspector 29 | implements BinaryObjectInspector { 30 | 31 | public PhoenixBinaryObjectInspector() { 32 | super(TypeInfoFactory.binaryTypeInfo); 33 | } 34 | 35 | @Override 36 | public Object copyObject(Object o) { 37 | byte[] clone = null; 38 | 39 | if (o != null) { 40 | byte[] source = (byte[]) o; 41 | clone = new byte[source.length]; 42 | System.arraycopy(source, 0, clone, 0, source.length); 43 | } 44 | 45 | return clone; 46 | } 47 | 48 | @Override 49 | public byte[] getPrimitiveJavaObject(Object o) { 50 | return (byte[]) o; 51 | } 52 | 53 | @Override 54 | public BytesWritable getPrimitiveWritableObject(Object o) { 55 | return new BytesWritable((byte[]) o); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.hive.serde2.io.ByteWritable; 23 | 24 | /** 25 | * ObjectInspector for byte type 26 | */ 27 | public class PhoenixByteObjectInspector extends AbstractPhoenixObjectInspector 28 | implements ByteObjectInspector { 29 | 30 | public PhoenixByteObjectInspector() { 31 | super(TypeInfoFactory.byteTypeInfo); 32 | } 33 | 34 | @Override 35 | public Object copyObject(Object o) { 36 | return o == null ? null : new Byte((Byte) o); 37 | } 38 | 39 | @Override 40 | public ByteWritable getPrimitiveWritableObject(Object o) { 41 | return new ByteWritable(get(o)); 42 | } 43 | 44 | @Override 45 | public byte get(Object o) { 46 | Byte value = null; 47 | 48 | if (o != null) { 49 | try { 50 | value = (Byte) o; 51 | } catch (Exception e) { 52 | logExceptionMessage(o, "BYTE"); 53 | } 54 | } 55 | 56 | return value; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixByteObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.hive.serde2.io.ByteWritable; 23 | 24 | /** 25 | * ObjectInspector for byte type 26 | */ 27 | public class PhoenixByteObjectInspector extends AbstractPhoenixObjectInspector 28 | implements ByteObjectInspector { 29 | 30 | public PhoenixByteObjectInspector() { 31 | super(TypeInfoFactory.byteTypeInfo); 32 | } 33 | 34 | @Override 35 | public Object copyObject(Object o) { 36 | return o == null ? null : new Byte((Byte) o); 37 | } 38 | 39 | @Override 40 | public ByteWritable getPrimitiveWritableObject(Object o) { 41 | return new ByteWritable(get(o)); 42 | } 43 | 44 | @Override 45 | public byte get(Object o) { 46 | Byte value = null; 47 | 48 | if (o != null) { 49 | try { 50 | value = (Byte) o; 51 | } catch (Exception e) { 52 | logExceptionMessage(o, "BYTE"); 53 | } 54 | } 55 | 56 | return value; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/scala/org/apache/phoenix/spark/SparkSqlContextFunctions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | package org.apache.phoenix.spark 15 | 16 | import org.apache.hadoop.conf.Configuration 17 | import org.apache.spark.sql.{DataFrame, SQLContext} 18 | 19 | @deprecated("Use the DataSource V2 API implementation (see PhoenixDataSource)") 20 | class SparkSqlContextFunctions(@transient val sqlContext: SQLContext) extends Serializable { 21 | /* 22 | This will return a Spark DataFrame, with Phoenix types converted Spark SQL catalyst types 23 | 24 | 'table' is the corresponding Phoenix table 25 | 'columns' is a sequence of columns to query 26 | 'predicate' is a set of statements to go after a WHERE clause, e.g. "TID = 123" 27 | 'zkUrl' is an optional Zookeeper URL to use to connect to Phoenix 28 | 'conf' is a Hadoop Configuration object. If zkUrl is not set, the "hbase.zookeeper.quorum" 29 | property will be used 30 | */ 31 | def phoenixTableAsDataFrame(table: String, columns: Seq[String], 32 | predicate: Option[String] = None, 33 | zkUrl: Option[String] = None, 34 | tenantId: Option[String] = None, 35 | conf: Configuration = new Configuration): DataFrame = { 36 | implicit val sparkSession = sqlContext.sparkSession 37 | val df = PhoenixDataFrameHelper.createDataFrame(table, zkUrl, tenantId, conf) 38 | val dfWithSelectColumns = PhoenixDataFrameHelper.withSelectExpr(columns, df) 39 | PhoenixDataFrameHelper.withWhereCondition(predicate, dfWithSelectColumns) 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/PhoenixRowKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive; 19 | 20 | import org.apache.hadoop.hive.ql.io.RecordIdentifier; 21 | 22 | import java.io.DataInput; 23 | import java.io.DataOutput; 24 | import java.io.IOException; 25 | import java.io.InputStream; 26 | import java.io.OutputStream; 27 | import java.util.Map; 28 | 29 | /** 30 | * Hive's RecordIdentifier implementation. 31 | */ 32 | 33 | public class PhoenixRowKey extends RecordIdentifier { 34 | 35 | private PrimaryKeyData rowKeyMap = PrimaryKeyData.EMPTY; 36 | 37 | public PhoenixRowKey() { 38 | 39 | } 40 | 41 | public void setRowKeyMap(Map rowKeyMap) { 42 | this.rowKeyMap = new PrimaryKeyData(rowKeyMap); 43 | } 44 | 45 | @Override 46 | public void write(DataOutput dataOutput) throws IOException { 47 | super.write(dataOutput); 48 | 49 | rowKeyMap.serialize((OutputStream) dataOutput); 50 | } 51 | 52 | @Override 53 | public void readFields(DataInput dataInput) throws IOException { 54 | super.readFields(dataInput); 55 | 56 | try { 57 | rowKeyMap = PrimaryKeyData.deserialize((InputStream) dataInput); 58 | } catch (ClassNotFoundException e) { 59 | throw new RuntimeException(e); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixBinaryObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.BytesWritable; 23 | 24 | /** 25 | * ObjectInspector for Binary type 26 | */ 27 | 28 | public class PhoenixBinaryObjectInspector extends AbstractPhoenixObjectInspector 29 | implements BinaryObjectInspector { 30 | 31 | public PhoenixBinaryObjectInspector() { 32 | super(TypeInfoFactory.binaryTypeInfo); 33 | } 34 | 35 | @Override 36 | public Object copyObject(Object o) { 37 | byte[] clone = null; 38 | 39 | if (o != null) { 40 | byte[] source = (byte[]) o; 41 | clone = new byte[source.length]; 42 | System.arraycopy(source, 0, clone, 0, source.length); 43 | } 44 | 45 | return clone; 46 | } 47 | 48 | @Override 49 | public byte[] getPrimitiveJavaObject(Object o) { 50 | return (byte[]) o; 51 | } 52 | 53 | @Override 54 | public BytesWritable getPrimitiveWritableObject(Object o) { 55 | return new BytesWritable((byte[]) o); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/PhoenixRowKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive; 19 | 20 | import org.apache.hadoop.hive.ql.io.RecordIdentifier; 21 | 22 | import java.io.DataInput; 23 | import java.io.DataOutput; 24 | import java.io.IOException; 25 | import java.io.InputStream; 26 | import java.io.OutputStream; 27 | import java.util.Map; 28 | 29 | /** 30 | * Hive's RecordIdentifier implementation. 31 | */ 32 | 33 | public class PhoenixRowKey extends RecordIdentifier { 34 | 35 | private PrimaryKeyData rowKeyMap = PrimaryKeyData.EMPTY; 36 | 37 | public PhoenixRowKey() { 38 | 39 | } 40 | 41 | public void setRowKeyMap(Map rowKeyMap) { 42 | this.rowKeyMap = new PrimaryKeyData(rowKeyMap); 43 | } 44 | 45 | @Override 46 | public void write(DataOutput dataOutput) throws IOException { 47 | super.write(dataOutput); 48 | 49 | rowKeyMap.serialize((OutputStream) dataOutput); 50 | } 51 | 52 | @Override 53 | public void readFields(DataInput dataInput) throws IOException { 54 | super.readFields(dataInput); 55 | 56 | try { 57 | rowKeyMap = PrimaryKeyData.deserialize((InputStream) dataInput); 58 | } catch (ClassNotFoundException e) { 59 | throw new RuntimeException(e); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.FloatWritable; 23 | 24 | /** 25 | * ObjectInspector for float type 26 | */ 27 | 28 | public class PhoenixFloatObjectInspector extends AbstractPhoenixObjectInspector 29 | implements FloatObjectInspector { 30 | 31 | public PhoenixFloatObjectInspector() { 32 | super(TypeInfoFactory.floatTypeInfo); 33 | } 34 | 35 | @Override 36 | public Object copyObject(Object o) { 37 | return o == null ? null : new Float((Float) o); 38 | } 39 | 40 | @Override 41 | public FloatWritable getPrimitiveWritableObject(Object o) { 42 | return new FloatWritable(get(o)); 43 | } 44 | 45 | @Override 46 | public float get(Object o) { 47 | Float value = null; 48 | 49 | if (o != null) { 50 | try { 51 | value = ((Float) o).floatValue(); 52 | } catch (Exception e) { 53 | logExceptionMessage(o, "LONG"); 54 | } 55 | } 56 | 57 | return value; 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixFloatObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.FloatWritable; 23 | 24 | /** 25 | * ObjectInspector for float type 26 | */ 27 | 28 | public class PhoenixFloatObjectInspector extends AbstractPhoenixObjectInspector 29 | implements FloatObjectInspector { 30 | 31 | public PhoenixFloatObjectInspector() { 32 | super(TypeInfoFactory.floatTypeInfo); 33 | } 34 | 35 | @Override 36 | public Object copyObject(Object o) { 37 | return o == null ? null : new Float((Float) o); 38 | } 39 | 40 | @Override 41 | public FloatWritable getPrimitiveWritableObject(Object o) { 42 | return new FloatWritable(get(o)); 43 | } 44 | 45 | @Override 46 | public float get(Object o) { 47 | Float value = null; 48 | 49 | if (o != null) { 50 | try { 51 | value = ((Float) o).floatValue(); 52 | } catch (Exception e) { 53 | logExceptionMessage(o, "LONG"); 54 | } 55 | } 56 | 57 | return value; 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/sql/connector/writer/PhoenixTestBatchWrite.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.writer; 19 | 20 | import org.apache.spark.sql.connector.write.DataWriterFactory; 21 | import org.apache.spark.sql.connector.write.LogicalWriteInfo; 22 | import org.apache.spark.sql.connector.write.PhysicalWriteInfo; 23 | import org.apache.spark.sql.connector.write.WriterCommitMessage; 24 | 25 | import java.util.Map; 26 | 27 | 28 | public class PhoenixTestBatchWrite extends PhoenixBatchWrite { 29 | public static int TOTAL_BATCHES_COMMITTED_COUNT = 0; 30 | PhoenixTestBatchWrite(LogicalWriteInfo writeInfo, Map options) { 31 | super(writeInfo, options); 32 | } 33 | 34 | // Override to return a test DataWriterFactory 35 | @Override 36 | public DataWriterFactory createBatchWriterFactory(PhysicalWriteInfo physicalWriteInfo) { 37 | return new PhoenixTestDataWriterFactory(getWriteInfo().schema(), getOptions()); 38 | } 39 | 40 | // Override to sum up the total number of batches committed across all executors 41 | @Override 42 | public void commit(WriterCommitMessage[] messages) { 43 | for (WriterCommitMessage message : messages) { 44 | TOTAL_BATCHES_COMMITTED_COUNT += Integer.parseInt(message.toString()); 45 | } 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /phoenix5-spark/src/it/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/scala/org/apache/phoenix/spark/DataFrameFunctions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | package org.apache.phoenix.spark 15 | 16 | import org.apache.hadoop.conf.Configuration 17 | import org.apache.phoenix.spark.PhoenixDataFrameHelper.phoenixConfig 18 | import org.apache.phoenix.spark.datasource.v2.PhoenixDataSource 19 | import org.apache.spark.sql.{DataFrame, SaveMode} 20 | 21 | @deprecated("Use the DataSource V2 API implementation (see PhoenixDataSource)") 22 | class DataFrameFunctions(data: DataFrame) extends Serializable { 23 | def saveToPhoenix(parameters: Map[String, String]): Unit = { 24 | saveToPhoenix( 25 | tableName = parameters("table"), 26 | zkUrl = parameters.get("zkUrl"), 27 | tenantId = parameters.get("TenantId"), 28 | skipNormalizingIdentifier = parameters.contains("skipNormalizingIdentifier") 29 | ) 30 | } 31 | 32 | def saveToPhoenix(tableName: String, 33 | conf: Configuration = new Configuration, 34 | zkUrl: Option[String] = None, 35 | tenantId: Option[String] = None, 36 | skipNormalizingIdentifier: Boolean = false): Unit = { 37 | data 38 | .write 39 | .format("phoenix") 40 | .mode(SaveMode.Overwrite) 41 | .option(PhoenixDataSource.TABLE, tableName) 42 | .option(PhoenixDataSource.JDBC_URL, zkUrl.orNull) 43 | .option(PhoenixDataSource.TENANT_ID, tenantId.orNull) 44 | .option(PhoenixDataSource.PHOENIX_CONFIGS, phoenixConfig(conf)) 45 | .option(PhoenixDataSource.SKIP_NORMALIZING_IDENTIFIER, skipNormalizingIdentifier) 46 | .save() 47 | 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/scala/org/apache/phoenix/spark/DataFrameFunctions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | package org.apache.phoenix.spark 15 | 16 | import org.apache.hadoop.conf.Configuration 17 | import org.apache.phoenix.spark.PhoenixDataFrameHelper.phoenixConfig 18 | import org.apache.phoenix.spark.sql.connector.PhoenixDataSource 19 | import org.apache.spark.sql.{DataFrame, SaveMode} 20 | 21 | @deprecated("Use the DataSource V2 API implementation (see PhoenixDataSource)") 22 | class DataFrameFunctions(data: DataFrame) extends Serializable { 23 | def saveToPhoenix(parameters: Map[String, String]): Unit = { 24 | saveToPhoenix( 25 | tableName = parameters("table"), 26 | zkUrl = parameters.get("zkUrl"), 27 | tenantId = parameters.get("TenantId"), 28 | skipNormalizingIdentifier = parameters.contains("skipNormalizingIdentifier") 29 | ) 30 | } 31 | 32 | def saveToPhoenix(tableName: String, 33 | conf: Configuration = new Configuration, 34 | zkUrl: Option[String] = None, 35 | tenantId: Option[String] = None, 36 | skipNormalizingIdentifier: Boolean = false): Unit = { 37 | data 38 | .write 39 | .format("phoenix") 40 | .mode(SaveMode.Append) 41 | .option(PhoenixDataSource.TABLE, tableName) 42 | .option(PhoenixDataSource.JDBC_URL, zkUrl.orNull) 43 | .option(PhoenixDataSource.TENANT_ID, tenantId.orNull) 44 | .option(PhoenixDataSource.PHOENIX_CONFIGS, phoenixConfig(conf)) 45 | .option(PhoenixDataSource.SKIP_NORMALIZING_IDENTIFIER, skipNormalizingIdentifier) 46 | .save() 47 | 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.hive.serde2.io.DoubleWritable; 23 | 24 | /** 25 | * ObjectInspector for double type 26 | */ 27 | public class PhoenixDoubleObjectInspector extends AbstractPhoenixObjectInspector 28 | implements DoubleObjectInspector { 29 | 30 | public PhoenixDoubleObjectInspector() { 31 | super(TypeInfoFactory.doubleTypeInfo); 32 | } 33 | 34 | @Override 35 | public Object copyObject(Object o) { 36 | return o == null ? null : new Double((Double) o); 37 | } 38 | 39 | @Override 40 | public DoubleWritable getPrimitiveWritableObject(Object o) { 41 | return new DoubleWritable(get(o)); 42 | } 43 | 44 | @Override 45 | public double get(Object o) { 46 | Double value = null; 47 | 48 | if (o != null) { 49 | try { 50 | value = ((Double) o).doubleValue(); 51 | } catch (Exception e) { 52 | logExceptionMessage(o, "LONG"); 53 | } 54 | } 55 | 56 | return value; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDoubleObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.hive.serde2.io.DoubleWritable; 23 | 24 | /** 25 | * ObjectInspector for double type 26 | */ 27 | public class PhoenixDoubleObjectInspector extends AbstractPhoenixObjectInspector 28 | implements DoubleObjectInspector { 29 | 30 | public PhoenixDoubleObjectInspector() { 31 | super(TypeInfoFactory.doubleTypeInfo); 32 | } 33 | 34 | @Override 35 | public Object copyObject(Object o) { 36 | return o == null ? null : new Double((Double) o); 37 | } 38 | 39 | @Override 40 | public DoubleWritable getPrimitiveWritableObject(Object o) { 41 | return new DoubleWritable(get(o)); 42 | } 43 | 44 | @Override 45 | public double get(Object o) { 46 | Double value = null; 47 | 48 | if (o != null) { 49 | try { 50 | value = ((Double) o).doubleValue(); 51 | } catch (Exception e) { 52 | logExceptionMessage(o, "LONG"); 53 | } 54 | } 55 | 56 | return value; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/java/org/apache/phoenix/spark/datasource/v2/reader/PhoenixTestingInputPartitionReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.reader; 19 | 20 | import org.apache.phoenix.mapreduce.PhoenixInputSplit; 21 | import org.apache.spark.SerializableWritable; 22 | import org.apache.spark.sql.catalyst.InternalRow; 23 | import org.apache.spark.sql.types.StructType; 24 | 25 | import java.util.Properties; 26 | 27 | public class PhoenixTestingInputPartitionReader extends PhoenixInputPartitionReader { 28 | 29 | // A test property which is used to modify the current row returned by the test input 30 | // partition reader in order to check properties passed from the driver to executors 31 | public static final String RETURN_NULL_CURR_ROW = "return.null.curr.row"; 32 | 33 | PhoenixTestingInputPartitionReader(PhoenixDataSourceReadOptions options, StructType schema, 34 | SerializableWritable phoenixInputSplit) { 35 | super(options, schema, phoenixInputSplit); 36 | } 37 | 38 | // Override to return null rather than the actual row based on a property passed to the executor 39 | @Override 40 | public InternalRow get() { 41 | Properties props = getOverriddenPropsFromOptions(); 42 | return Boolean.valueOf(props.getProperty(RETURN_NULL_CURR_ROW)) ? null : super.get(); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/PhoenixRow.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive; 19 | 20 | import org.apache.hadoop.hive.serde2.StructObject; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | /** 27 | * Implementation for Hive SerDe StructObject 28 | */ 29 | public class PhoenixRow implements StructObject { 30 | 31 | private List columnList; 32 | private Map resultRowMap; 33 | 34 | public PhoenixRow(List columnList) { 35 | this.columnList = columnList; 36 | } 37 | 38 | public PhoenixRow setResultRowMap(Map resultRowMap) { 39 | this.resultRowMap = resultRowMap; 40 | return this; 41 | } 42 | 43 | /* (non-Javadoc) 44 | * @see org.apache.hadoop.hive.serde2.StructObject#getField(int) 45 | */ 46 | @Override 47 | public Object getField(int fieldID) { 48 | return resultRowMap.get(columnList.get(fieldID)); 49 | } 50 | 51 | /* (non-Javadoc) 52 | * @see org.apache.hadoop.hive.serde2.StructObject#getFieldsAsList() 53 | */ 54 | @Override 55 | public List getFieldsAsList() { 56 | return new ArrayList<>(resultRowMap.values()); 57 | } 58 | 59 | 60 | @Override 61 | public String toString() { 62 | return resultRowMap.toString(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.IntWritable; 23 | 24 | public class PhoenixIntObjectInspector extends AbstractPhoenixObjectInspector 25 | implements IntObjectInspector { 26 | 27 | public PhoenixIntObjectInspector() { 28 | super(TypeInfoFactory.intTypeInfo); 29 | } 30 | 31 | @Override 32 | public Object copyObject(Object o) { 33 | return o == null ? null : new Integer((Integer) o); 34 | } 35 | 36 | @Override 37 | public Category getCategory() { 38 | return Category.PRIMITIVE; 39 | } 40 | 41 | @Override 42 | public IntWritable getPrimitiveWritableObject(Object o) { 43 | return new IntWritable(get(o)); 44 | } 45 | 46 | @Override 47 | public int get(Object o) { 48 | Integer value = null; 49 | 50 | if (o != null) { 51 | try { 52 | value = ((Integer) o).intValue(); 53 | } catch (Exception e) { 54 | logExceptionMessage(o, "INT"); 55 | } 56 | } 57 | 58 | return value; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/PhoenixRow.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive; 19 | 20 | import org.apache.hadoop.hive.serde2.StructObject; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | /** 27 | * Implementation for Hive SerDe StructObject 28 | */ 29 | public class PhoenixRow implements StructObject { 30 | 31 | private List columnList; 32 | private Map resultRowMap; 33 | 34 | public PhoenixRow(List columnList) { 35 | this.columnList = columnList; 36 | } 37 | 38 | public PhoenixRow setResultRowMap(Map resultRowMap) { 39 | this.resultRowMap = resultRowMap; 40 | return this; 41 | } 42 | 43 | /* (non-Javadoc) 44 | * @see org.apache.hadoop.hive.serde2.StructObject#getField(int) 45 | */ 46 | @Override 47 | public Object getField(int fieldID) { 48 | return resultRowMap.get(columnList.get(fieldID)); 49 | } 50 | 51 | /* (non-Javadoc) 52 | * @see org.apache.hadoop.hive.serde2.StructObject#getFieldsAsList() 53 | */ 54 | @Override 55 | public List getFieldsAsList() { 56 | return new ArrayList<>(resultRowMap.values()); 57 | } 58 | 59 | 60 | @Override 61 | public String toString() { 62 | return resultRowMap.toString(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixIntObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 22 | import org.apache.hadoop.io.IntWritable; 23 | 24 | public class PhoenixIntObjectInspector extends AbstractPhoenixObjectInspector 25 | implements IntObjectInspector { 26 | 27 | public PhoenixIntObjectInspector() { 28 | super(TypeInfoFactory.intTypeInfo); 29 | } 30 | 31 | @Override 32 | public Object copyObject(Object o) { 33 | return o == null ? null : new Integer((Integer) o); 34 | } 35 | 36 | @Override 37 | public Category getCategory() { 38 | return Category.PRIMITIVE; 39 | } 40 | 41 | @Override 42 | public IntWritable getPrimitiveWritableObject(Object o) { 43 | return new IntWritable(get(o)); 44 | } 45 | 46 | @Override 47 | public int get(Object o) { 48 | Integer value = null; 49 | 50 | if (o != null) { 51 | try { 52 | value = ((Integer) o).intValue(); 53 | } catch (Exception e) { 54 | logExceptionMessage(o, "INT"); 55 | } 56 | } 57 | 58 | return value; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /phoenix5-spark3/src/main/java/org/apache/phoenix/spark/sql/connector/reader/PhoenixPartitionReadFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.sql.connector.reader; 19 | 20 | import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting; 21 | import org.apache.spark.sql.catalyst.InternalRow; 22 | import org.apache.spark.sql.connector.read.InputPartition; 23 | import org.apache.spark.sql.connector.read.PartitionReader; 24 | import org.apache.spark.sql.connector.read.PartitionReaderFactory; 25 | import org.apache.spark.sql.types.StructType; 26 | 27 | public class PhoenixPartitionReadFactory implements PartitionReaderFactory { 28 | 29 | private final StructType schema; 30 | PhoenixDataSourceReadOptions options; 31 | 32 | PhoenixPartitionReadFactory(PhoenixDataSourceReadOptions phoenixDataSourceOptions, StructType schema) { 33 | this.schema = schema; 34 | this.options = phoenixDataSourceOptions; 35 | } 36 | 37 | @VisibleForTesting 38 | PhoenixDataSourceReadOptions getOptions() { 39 | return options; 40 | } 41 | 42 | @VisibleForTesting 43 | StructType readSchema() { 44 | return schema; 45 | } 46 | 47 | @Override 48 | public PartitionReader createReader(InputPartition partition) { 49 | return new PhoenixPartitionReader(options, schema, (PhoenixInputPartition) partition); 50 | } 51 | } -------------------------------------------------------------------------------- /src/main/config/checkstyle/suppressions.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /phoenix5-hive/src/it/java/org/apache/hadoop/hive/ql/security/DummyAuthenticator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hive.ql.security; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import org.apache.hadoop.conf.Configuration; 24 | import org.apache.hadoop.hive.ql.metadata.HiveException; 25 | import org.apache.hadoop.hive.ql.session.SessionState; 26 | 27 | public class DummyAuthenticator implements HiveAuthenticationProvider { 28 | 29 | private final List groupNames; 30 | private final String userName; 31 | private Configuration conf; 32 | 33 | public DummyAuthenticator() { 34 | this.groupNames = new ArrayList(); 35 | groupNames.add("hive_test_group1"); 36 | groupNames.add("hive_test_group2"); 37 | userName = "hive_test_user"; 38 | } 39 | 40 | @Override 41 | public void destroy() throws HiveException{ 42 | return; 43 | } 44 | 45 | @Override 46 | public List getGroupNames() { 47 | return groupNames; 48 | } 49 | 50 | @Override 51 | public String getUserName() { 52 | return userName; 53 | } 54 | 55 | @Override 56 | public void setConf(Configuration conf) { 57 | this.conf = conf; 58 | } 59 | 60 | @Override 61 | public Configuration getConf() { 62 | return this.conf; 63 | } 64 | 65 | @Override 66 | public void setSessionState(SessionState ss) { 67 | //no op 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/scheduled/QTestScheduledQueryServiceProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hive.ql.scheduled; 19 | 20 | import org.apache.hadoop.hive.conf.HiveConf; 21 | import org.apache.hadoop.hive.ql.QTestUtil; 22 | import org.apache.hadoop.hive.ql.qoption.QTestOptionHandler; 23 | 24 | /** 25 | * Adding this qtestoption enables the scheduled query service. 26 | */ 27 | public class QTestScheduledQueryServiceProvider implements QTestOptionHandler { 28 | 29 | private boolean enabled; 30 | private ScheduledQueryExecutionService service; 31 | 32 | public QTestScheduledQueryServiceProvider(HiveConf conf) { 33 | conf.setVar(HiveConf.ConfVars.HIVE_SCHEDULED_QUERIES_EXECUTOR_IDLE_SLEEP_TIME, "1s"); 34 | conf.setVar(HiveConf.ConfVars.HIVE_SCHEDULED_QUERIES_EXECUTOR_PROGRESS_REPORT_INTERVAL, "1s"); 35 | } 36 | 37 | @Override 38 | public void processArguments(String arguments) { 39 | enabled = true; 40 | } 41 | 42 | @Override 43 | public void beforeTest(QTestUtil qt) throws Exception { 44 | if (enabled) { 45 | service = ScheduledQueryExecutionService.startScheduledQueryExecutorService(qt.getConf()); 46 | } 47 | } 48 | 49 | @Override 50 | public void afterTest(QTestUtil qt) throws Exception { 51 | if(service != null) { 52 | service.close(); 53 | } 54 | service = null; 55 | enabled = false; 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/security/DummyAuthenticator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hive.ql.security; 19 | 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | import org.apache.hadoop.conf.Configuration; 24 | import org.apache.hadoop.hive.ql.metadata.HiveException; 25 | import org.apache.hadoop.hive.ql.session.ISessionAuthState; 26 | 27 | public class DummyAuthenticator implements HiveAuthenticationProvider { 28 | 29 | private final List groupNames; 30 | private final String userName; 31 | private Configuration conf; 32 | 33 | public DummyAuthenticator() { 34 | this.groupNames = new ArrayList(); 35 | groupNames.add("hive_test_group1"); 36 | groupNames.add("hive_test_group2"); 37 | userName = "hive_test_user"; 38 | } 39 | 40 | @Override 41 | public void destroy() throws HiveException{ 42 | return; 43 | } 44 | 45 | @Override 46 | public List getGroupNames() { 47 | return groupNames; 48 | } 49 | 50 | @Override 51 | public String getUserName() { 52 | return userName; 53 | } 54 | 55 | @Override 56 | public void setConf(Configuration conf) { 57 | this.conf = conf; 58 | } 59 | 60 | @Override 61 | public Configuration getConf() { 62 | return this.conf; 63 | } 64 | 65 | @Override 66 | public void setSessionState(ISessionAuthState ss) { 67 | //no op 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml: -------------------------------------------------------------------------------- 1 | 2 | 22 | 23 | 24 | 25 | ${project.basedir}/../phoenix5-spark/target 26 | / 27 | 28 | phoenix5-spark-${project.version}.jar 29 | phoenix5-spark.jar 30 | 31 | 32 | 33 | ${project.basedir}/../phoenix5-spark3/target 34 | / 35 | 36 | phoenix5-spark3-${project.version}.jar 37 | phoenix5-spark3.jar 38 | 39 | 40 | 41 | ${project.basedir}/../phoenix5-hive/target 42 | / 43 | 44 | phoenix5-hive-${project.version}.jar 45 | phoenix5-hive.jar 46 | 47 | 48 | 49 | ${project.basedir}/../phoenix5-hive4/target 50 | / 51 | 52 | phoenix5-hive4-${project.version}.jar 53 | phoenix5-hive4.jar 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.common.type.HiveChar; 21 | import org.apache.hadoop.hive.serde2.io.HiveCharWritable; 22 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector; 23 | import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; 24 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 25 | 26 | /** 27 | * ObjectInspector for char type 28 | */ 29 | public class PhoenixCharObjectInspector extends AbstractPhoenixObjectInspector 30 | implements HiveCharObjectInspector { 31 | 32 | public PhoenixCharObjectInspector() { 33 | this(TypeInfoFactory.charTypeInfo); 34 | } 35 | 36 | public PhoenixCharObjectInspector(PrimitiveTypeInfo type) { 37 | super(type); 38 | } 39 | 40 | @Override 41 | public Object copyObject(Object o) { 42 | return o == null ? null : new String((String) o); 43 | } 44 | 45 | @Override 46 | public HiveCharWritable getPrimitiveWritableObject(Object o) { 47 | return new HiveCharWritable(getPrimitiveJavaObject(o)); 48 | } 49 | 50 | @Override 51 | public HiveChar getPrimitiveJavaObject(Object o) { 52 | String value = (String) o; 53 | return new HiveChar(value, value.length()); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixCharObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.common.type.HiveChar; 21 | import org.apache.hadoop.hive.serde2.io.HiveCharWritable; 22 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector; 23 | import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; 24 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 25 | 26 | /** 27 | * ObjectInspector for char type 28 | */ 29 | public class PhoenixCharObjectInspector extends AbstractPhoenixObjectInspector 30 | implements HiveCharObjectInspector { 31 | 32 | public PhoenixCharObjectInspector() { 33 | this(TypeInfoFactory.charTypeInfo); 34 | } 35 | 36 | public PhoenixCharObjectInspector(PrimitiveTypeInfo type) { 37 | super(type); 38 | } 39 | 40 | @Override 41 | public Object copyObject(Object o) { 42 | return o == null ? null : new String((String) o); 43 | } 44 | 45 | @Override 46 | public HiveCharWritable getPrimitiveWritableObject(Object o) { 47 | return new HiveCharWritable(getPrimitiveJavaObject(o)); 48 | } 49 | 50 | @Override 51 | public HiveChar getPrimitiveJavaObject(Object o) { 52 | String value = (String) o; 53 | return new HiveChar(value, value.length()); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/qoption/QTestAuthorizerHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.hive.ql.qoption; 20 | 21 | import org.apache.hadoop.hive.ql.QTestUtil; 22 | import org.apache.hadoop.hive.ql.session.SessionState; 23 | 24 | /** 25 | * QTest authorizer option 26 | * 27 | * Enables authorization for the qtest. 28 | * 29 | * Example: 30 | * --! qt:authorizer 31 | */ 32 | public class QTestAuthorizerHandler implements QTestOptionHandler { 33 | private boolean enabled; 34 | 35 | @Override 36 | public void processArguments(String arguments) { 37 | enabled = true; 38 | } 39 | 40 | @Override 41 | public void beforeTest(QTestUtil qt) throws Exception { 42 | if (enabled) { 43 | qt.getConf().set("hive.test.authz.sstd.hs2.mode", "true"); 44 | qt.getConf().set("hive.security.authorization.manager", 45 | "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest"); 46 | qt.getConf().set("hive.security.authenticator.manager", 47 | "org.apache.hadoop.hive.ql.security.SessionStateConfigUserAuthenticator"); 48 | qt.getConf().set("hive.security.authorization.enabled", "true"); 49 | SessionState.get().setAuthenticator(null); 50 | SessionState.get().setAuthorizer(null); 51 | } 52 | } 53 | 54 | @Override 55 | public void afterTest(QTestUtil qt) throws Exception { 56 | enabled = false; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.hive.ql.qoption; 20 | 21 | import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; 22 | import org.apache.hadoop.hive.ql.QTestUtil; 23 | import org.apache.hive.testutils.HiveTestEnvSetup; 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | /** 28 | * QTest sysdb directive handler 29 | * 30 | * Loads the sysdb schema prior to running the test. 31 | * 32 | * Example: 33 | * --! qt:sysdb 34 | * 35 | */ 36 | public class QTestSysDbHandler implements QTestOptionHandler { 37 | private static final Logger LOG = LoggerFactory.getLogger(QTestSysDbHandler.class.getName()); 38 | private boolean enabled; 39 | 40 | @Override 41 | public void processArguments(String arguments) { 42 | enabled = true; 43 | } 44 | 45 | @Override 46 | public void beforeTest(QTestUtil qt) throws Exception { 47 | if (enabled) { 48 | String schemaVersion = MetaStoreSchemaInfoFactory.get(qt.getConf()).getHiveSchemaVersion(); 49 | String stsdbPath = 50 | HiveTestEnvSetup.HIVE_ROOT + "/metastore/scripts/upgrade/hive/hive-schema-" + schemaVersion + ".hive.sql"; 51 | qt.getCliDriver().processLine("source " + stsdbPath); 52 | qt.getCliDriver().processLine("use default"); 53 | } 54 | } 55 | 56 | @Override 57 | public void afterTest(QTestUtil qt) throws Exception { 58 | enabled = false; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /dev/gather_machine_environment.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | # SHELLDOC-IGNORE 20 | 21 | set -e 22 | function usage { 23 | echo "Usage: ${0} /path/for/output/dir" 24 | echo "" 25 | echo " Gather info about a build machine that test harnesses should poll before running." 26 | echo " presumes you'll then archive the passed output dir." 27 | 28 | exit 1 29 | } 30 | 31 | if [ "$#" -lt 1 ]; then 32 | usage 33 | fi 34 | 35 | 36 | declare output=$1 37 | 38 | if [ ! -d "${output}" ] || [ ! -w "${output}" ]; then 39 | echo "Specified output directory must exist and be writable." >&2 40 | exit 1 41 | fi 42 | 43 | echo "getting machine specs, find in ${BUILD_URL}/artifact/${output}/" 44 | echo "JAVA_HOME: ${JAVA_HOME}" >"${output}/java_home" 2>&1 || true 45 | ls -l "${JAVA_HOME}" >"${output}/java_home_ls" 2>&1 || true 46 | echo "MAVEN_HOME: ${MAVEN_HOME}" >"${output}/mvn_home" 2>&1 || true 47 | mvn --offline --version >"${output}/mvn_version" 2>&1 || true 48 | cat /proc/cpuinfo >"${output}/cpuinfo" 2>&1 || true 49 | cat /proc/meminfo >"${output}/meminfo" 2>&1 || true 50 | cat /proc/diskstats >"${output}/diskstats" 2>&1 || true 51 | cat /sys/block/sda/stat >"${output}/sys-block-sda-stat" 2>&1 || true 52 | df -h >"${output}/df-h" 2>&1 || true 53 | ps -Aww >"${output}/ps-Aww" 2>&1 || true 54 | ifconfig -a >"${output}/ifconfig-a" 2>&1 || true 55 | lsblk -ta >"${output}/lsblk-ta" 2>&1 || true 56 | lsblk -fa >"${output}/lsblk-fa" 2>&1 || true 57 | ulimit -a >"${output}/ulimit-a" 2>&1 || true 58 | uptime >"${output}/uptime" 2>&1 || true 59 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/java/org/apache/phoenix/spark/datasource/v2/writer/PhoenixTestingDataSourceWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.writer; 19 | 20 | import org.apache.spark.sql.SaveMode; 21 | import org.apache.spark.sql.catalyst.InternalRow; 22 | import org.apache.spark.sql.sources.v2.DataSourceOptions; 23 | import org.apache.spark.sql.sources.v2.writer.DataWriterFactory; 24 | import org.apache.spark.sql.sources.v2.writer.WriterCommitMessage; 25 | import org.apache.spark.sql.types.StructType; 26 | 27 | public class PhoenixTestingDataSourceWriter extends PhoenixDataSourceWriter { 28 | 29 | // Used to keep track of the total number of batches committed across all executors 30 | public static int TOTAL_BATCHES_COMMITTED_COUNT = 0; 31 | 32 | public PhoenixTestingDataSourceWriter(SaveMode mode, StructType schema, 33 | DataSourceOptions options) { 34 | super(mode, schema, options); 35 | } 36 | 37 | // Override to return a test DataWriterFactory 38 | @Override 39 | public DataWriterFactory createWriterFactory() { 40 | return new PhoenixTestingDataWriterFactory(getOptions()); 41 | } 42 | 43 | // Override to sum up the total number of batches committed across all executors 44 | @Override 45 | public void commit(WriterCommitMessage[] messages) { 46 | for (WriterCommitMessage message : messages) { 47 | TOTAL_BATCHES_COMMITTED_COUNT += Integer.parseInt(message.toString()); 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /phoenix5-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | import static org.junit.Assert.assertFalse; 22 | import static org.junit.Assert.assertTrue; 23 | import static org.junit.Assert.fail; 24 | 25 | import java.sql.Connection; 26 | import java.sql.ResultSet; 27 | import java.sql.SQLException; 28 | 29 | import org.apache.phoenix.end2end.ParallelStatsDisabledTest; 30 | import org.apache.phoenix.end2end.salted.BaseSaltedTableIT; 31 | import org.apache.phoenix.util.QueryBuilder; 32 | import org.junit.experimental.categories.Category; 33 | 34 | @Category(ParallelStatsDisabledTest.class) 35 | public class SaltedTableIT extends BaseSaltedTableIT { 36 | 37 | @Override 38 | protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder, 39 | String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) { 40 | ResultSet rs = null; 41 | try { 42 | rs = executeQuery(conn, queryBuilder); 43 | fail(); 44 | } 45 | catch(Exception e) { 46 | assertTrue(e.getMessage().contains(expectedSparkExceptionMsg)); 47 | } 48 | return rs; 49 | } 50 | 51 | @Override 52 | protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException { 53 | return SparkUtil.executeQuery(conn, queryBuilder, getUrl(), config); 54 | } 55 | 56 | } -------------------------------------------------------------------------------- /phoenix5-spark3/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark; 19 | 20 | import static org.junit.Assert.assertEquals; 21 | import static org.junit.Assert.assertFalse; 22 | import static org.junit.Assert.assertTrue; 23 | import static org.junit.Assert.fail; 24 | 25 | import java.sql.Connection; 26 | import java.sql.ResultSet; 27 | import java.sql.SQLException; 28 | 29 | import org.apache.phoenix.end2end.ParallelStatsDisabledTest; 30 | import org.apache.phoenix.end2end.salted.BaseSaltedTableIT; 31 | import org.apache.phoenix.util.QueryBuilder; 32 | import org.junit.experimental.categories.Category; 33 | 34 | @Category(ParallelStatsDisabledTest.class) 35 | public class SaltedTableIT extends BaseSaltedTableIT { 36 | 37 | @Override 38 | protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder, 39 | String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) { 40 | ResultSet rs = null; 41 | try { 42 | rs = executeQuery(conn, queryBuilder); 43 | fail(); 44 | } 45 | catch(Exception e) { 46 | assertTrue(e.getMessage().contains(expectedSparkExceptionMsg)); 47 | } 48 | return rs; 49 | } 50 | 51 | @Override 52 | protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException { 53 | return SparkUtil.executeQuery(conn, queryBuilder, getUrl(), config); 54 | } 55 | 56 | } -------------------------------------------------------------------------------- /phoenix5-connectors-assembly/src/build/components/phoenix5-shaded-jars.xml: -------------------------------------------------------------------------------- 1 | 2 | 22 | 23 | 24 | 25 | ${project.basedir}/../phoenix5-spark-shaded/target 26 | / 27 | 28 | phoenix5-spark-shaded-${project.version}.jar 29 | phoenix5-spark-shaded.jar 30 | 31 | 32 | 33 | ${project.basedir}/../phoenix5-spark3-shaded/target 34 | / 35 | 36 | phoenix5-spark3-shaded-${project.version}.jar 37 | phoenix5-spark3-shaded.jar 38 | 39 | 40 | 41 | ${project.basedir}/../phoenix5-hive-shaded/target 42 | / 43 | 44 | phoenix5-hive-shaded-${project.version}.jar 45 | phoenix5-hive-shaded.jar 46 | 47 | 48 | 49 | ${project.basedir}/../phoenix5-hive4-shaded/target 50 | / 51 | 52 | phoenix5-hive4-shaded-${project.version}.jar 53 | phoenix5-hive4-shaded.jar 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /phoenix5-spark/src/test/java/org/apache/phoenix/spark/datasource/v2/PhoenixTestingDataSource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2; 19 | 20 | import org.apache.phoenix.spark.datasource.v2.reader.PhoenixTestingDataSourceReader; 21 | import org.apache.phoenix.spark.datasource.v2.writer.PhoenixTestingDataSourceWriter; 22 | import org.apache.spark.sql.SaveMode; 23 | import org.apache.spark.sql.sources.v2.DataSourceOptions; 24 | import org.apache.spark.sql.sources.v2.reader.DataSourceReader; 25 | import org.apache.spark.sql.sources.v2.writer.DataSourceWriter; 26 | import org.apache.spark.sql.types.StructType; 27 | 28 | import java.util.Optional; 29 | 30 | public class PhoenixTestingDataSource extends PhoenixDataSource { 31 | 32 | public static final String TEST_SOURCE = 33 | "org.apache.phoenix.spark.datasource.v2.PhoenixTestingDataSource"; 34 | 35 | // Override to return a test DataSourceReader 36 | @Override 37 | public DataSourceReader createReader(DataSourceOptions options) { 38 | return new PhoenixTestingDataSourceReader(options); 39 | } 40 | 41 | // Override to return a test DataSourceWriter 42 | @Override 43 | public Optional createWriter(String writeUUID, StructType schema, 44 | SaveMode mode, DataSourceOptions options) { 45 | return Optional.of(new PhoenixTestingDataSourceWriter(mode, schema, options)); 46 | } 47 | 48 | @Override 49 | public String shortName() { 50 | return "phoenixTesting"; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/AbstractPhoenixObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.AbstractPrimitiveLazyObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; 22 | import org.apache.hadoop.io.Writable; 23 | 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | /** 28 | * AbstractPhoenixObjectInspector for a LazyPrimitive object 29 | */ 30 | public abstract class AbstractPhoenixObjectInspector 31 | extends AbstractPrimitiveLazyObjectInspector { 32 | 33 | private Logger log; 34 | 35 | public AbstractPhoenixObjectInspector() { 36 | super(); 37 | 38 | log = LoggerFactory.getLogger(getClass()); 39 | } 40 | 41 | protected AbstractPhoenixObjectInspector(PrimitiveTypeInfo typeInfo) { 42 | super(typeInfo); 43 | 44 | log = LoggerFactory.getLogger(getClass()); 45 | } 46 | 47 | @Override 48 | public Object getPrimitiveJavaObject(Object o) { 49 | return o == null ? null : o; 50 | } 51 | 52 | public void logExceptionMessage(Object value, String dataType) { 53 | if (log.isDebugEnabled()) { 54 | log.debug("Data not in the " + dataType + " data type range so converted to null. " + 55 | "Given data is :" 56 | + value.toString(), new Exception("For debugging purposes")); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/AbstractPhoenixObjectInspector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.AbstractPrimitiveLazyObjectInspector; 21 | import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; 22 | import org.apache.hadoop.io.Writable; 23 | 24 | import org.slf4j.Logger; 25 | import org.slf4j.LoggerFactory; 26 | 27 | /** 28 | * AbstractPhoenixObjectInspector for a LazyPrimitive object 29 | */ 30 | public abstract class AbstractPhoenixObjectInspector 31 | extends AbstractPrimitiveLazyObjectInspector { 32 | 33 | private Logger log; 34 | 35 | public AbstractPhoenixObjectInspector() { 36 | super(); 37 | 38 | log = LoggerFactory.getLogger(getClass()); 39 | } 40 | 41 | protected AbstractPhoenixObjectInspector(PrimitiveTypeInfo typeInfo) { 42 | super(typeInfo); 43 | 44 | log = LoggerFactory.getLogger(getClass()); 45 | } 46 | 47 | @Override 48 | public Object getPrimitiveJavaObject(Object o) { 49 | return o == null ? null : o; 50 | } 51 | 52 | public void logExceptionMessage(Object value, String dataType) { 53 | if (log.isDebugEnabled()) { 54 | log.debug("Data not in the " + dataType + " data type range so converted to null. " + 55 | "Given data is :" 56 | + value.toString(), new Exception("For debugging purposes")); 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /phoenix5-hive/src/it/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.hive.ql; 20 | 21 | /** 22 | * Standard output and return code of a process executed during the qtests. 23 | */ 24 | public class QTestProcessExecResult { 25 | 26 | private static final String TRUNCATED_OUTPUT = "Output was too long and had to be truncated..."; 27 | private static final short MAX_OUTPUT_CHAR_LENGTH = 2000; 28 | 29 | private final int returnCode; 30 | private final String standardOut; 31 | 32 | QTestProcessExecResult(int code, String output) { 33 | this.returnCode = code; 34 | this.standardOut = truncatefNeeded(output); 35 | } 36 | 37 | /** 38 | * @return executed process return code 39 | */ 40 | public int getReturnCode() { 41 | return this.returnCode; 42 | } 43 | 44 | /** 45 | * @return output captured from stdout while process was executing 46 | */ 47 | public String getCapturedOutput() { 48 | return this.standardOut; 49 | } 50 | 51 | public static QTestProcessExecResult create(int code, String output) { 52 | return new QTestProcessExecResult(code, output); 53 | } 54 | 55 | public static QTestProcessExecResult createWithoutOutput(int code) { 56 | return new QTestProcessExecResult(code, ""); 57 | } 58 | 59 | private String truncatefNeeded(String orig) { 60 | if (orig.length() > MAX_OUTPUT_CHAR_LENGTH) { 61 | return orig.substring(0, MAX_OUTPUT_CHAR_LENGTH) + "\r\n" + TRUNCATED_OUTPUT; 62 | } else { 63 | return orig; 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/QTestProcessExecResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.hadoop.hive.ql; 20 | 21 | /** 22 | * Standard output and return code of a process executed during the qtests. 23 | */ 24 | public class QTestProcessExecResult { 25 | 26 | private static final String TRUNCATED_OUTPUT = "Output was too long and had to be truncated..."; 27 | private static final short MAX_OUTPUT_CHAR_LENGTH = 2000; 28 | 29 | private final int returnCode; 30 | private final String standardOut; 31 | 32 | QTestProcessExecResult(int code, String output) { 33 | this.returnCode = code; 34 | this.standardOut = truncatefNeeded(output); 35 | } 36 | 37 | /** 38 | * @return executed process return code 39 | */ 40 | public int getReturnCode() { 41 | return this.returnCode; 42 | } 43 | 44 | /** 45 | * @return output captured from stdout while process was executing 46 | */ 47 | public String getCapturedOutput() { 48 | return this.standardOut; 49 | } 50 | 51 | public static QTestProcessExecResult create(int code, String output) { 52 | return new QTestProcessExecResult(code, output); 53 | } 54 | 55 | public static QTestProcessExecResult createWithoutOutput(int code) { 56 | return new QTestProcessExecResult(code, ""); 57 | } 58 | 59 | private String truncatefNeeded(String orig) { 60 | if (orig.length() > MAX_OUTPUT_CHAR_LENGTH) { 61 | return orig.substring(0, MAX_OUTPUT_CHAR_LENGTH) + "\r\n" + TRUNCATED_OUTPUT; 62 | } else { 63 | return orig; 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/java/org/apache/phoenix/spark/datasource/v2/reader/PhoenixInputPartition.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.reader; 19 | 20 | import org.apache.phoenix.mapreduce.PhoenixInputSplit; 21 | import org.apache.spark.SerializableWritable; 22 | import org.apache.spark.sql.catalyst.InternalRow; 23 | import org.apache.spark.sql.sources.v2.reader.InputPartition; 24 | import org.apache.spark.sql.sources.v2.reader.InputPartitionReader; 25 | import org.apache.spark.sql.types.StructType; 26 | 27 | public class PhoenixInputPartition implements InputPartition { 28 | 29 | private final SerializableWritable phoenixInputSplit; 30 | private final StructType schema; 31 | private final PhoenixDataSourceReadOptions options; 32 | 33 | PhoenixInputPartition(PhoenixDataSourceReadOptions options, StructType schema, PhoenixInputSplit phoenixInputSplit) { 34 | this.phoenixInputSplit = new SerializableWritable<>(phoenixInputSplit); 35 | this.schema = schema; 36 | this.options = options; 37 | } 38 | 39 | PhoenixDataSourceReadOptions getOptions() { 40 | return options; 41 | } 42 | 43 | StructType getSchema() { 44 | return schema; 45 | } 46 | 47 | SerializableWritable getPhoenixInputSplit() { 48 | return phoenixInputSplit; 49 | } 50 | 51 | @Override 52 | public InputPartitionReader createPartitionReader() { 53 | return new PhoenixInputPartitionReader(options, schema, phoenixInputSplit); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/it/java/org/apache/hadoop/hive/ql/qoption/QTestTimezoneHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.hadoop.hive.ql.qoption; 19 | 20 | import com.google.common.base.Strings; 21 | import java.util.TimeZone; 22 | import org.apache.hadoop.hive.ql.QTestUtil; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * QTest custom timezone handler 28 | * 29 | * Enables a custom timezone for the test. 30 | * 31 | * Example: 32 | * --! qt:timezone:Asia/Singapore 33 | * 34 | */ 35 | public class QTestTimezoneHandler implements QTestOptionHandler { 36 | 37 | private static final Logger LOG = LoggerFactory.getLogger(QTestTimezoneHandler.class); 38 | private boolean enabled = false; 39 | private TimeZone originalTimeZone; 40 | private TimeZone newTimeZone; 41 | 42 | @Override 43 | public void processArguments(String arguments) { 44 | if (Strings.isNullOrEmpty(arguments)) { 45 | throw new RuntimeException("illegal timezone arg: " + arguments); 46 | } 47 | originalTimeZone = TimeZone.getDefault(); 48 | newTimeZone = TimeZone.getTimeZone(arguments); 49 | LOG.info("Enabling timezone change: {} => {}", originalTimeZone, newTimeZone); 50 | enabled = true; 51 | } 52 | 53 | @Override 54 | public void beforeTest(QTestUtil qt) throws Exception { 55 | if (enabled) { 56 | TimeZone.setDefault(newTimeZone); 57 | } 58 | } 59 | 60 | @Override 61 | public void afterTest(QTestUtil qt) throws Exception { 62 | if (enabled) { 63 | TimeZone.setDefault(originalTimeZone); 64 | enabled = false; 65 | } 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /BUILDING.md: -------------------------------------------------------------------------------- 1 | 17 | 18 | ![logo](https://phoenix.apache.org/images/phoenix-logo-small.png) 19 | 20 | [Apache Phoenix](http://phoenix.apache.org/) enables OLTP and operational analytics in Hadoop for low latency applications. Visit the Apache Phoenix website [here](http://phoenix.apache.org/). This repo contains connectors for third party libraries to access data stored in Phoenix/HBase. 21 | 22 | Copyright ©2019 [Apache Software Foundation](http://www.apache.org/). All Rights Reserved. 23 | 24 | ## Building Apache Phoenix Connectors 25 | 26 | This repository will build jars for the different Phoenix connectors. 27 | 28 | ``` 29 | $ mvn package 30 | ``` 31 | 32 | ### Building against specific Phoenix version 33 | To build a release of Phoenix Connectors which packages a specific version of Phoenix, specify the `phoenix.version` system property to indicate a specific Phoenix version. 34 | 35 | 36 | When specifying `phoenix.version`, also specify the HBase version to be used 37 | by the corresponding `hbase-two.version` system property. 38 | Similarly you can override the `hadoop-three.version` system properties if necessary. 39 | 40 | ``` 41 | $ mvn package -Dphoenix-five.version=5.1.0-SNAPSHOT 42 | ``` 43 | 44 | ### Running integration tests 45 | 46 | `mvn package` will run the unit tests while building, but it will not run the integration test suite. 47 | 48 | The IT suite is run when executing `mvn install` or `mvn verify`. The Phoenix version specified 49 | with `phoenix.version` is used for running the integration tests. 50 | 51 | ``` 52 | $ mvn install -Dphoenix-five.version=5.1.0-SNAPSHOT -Dhbase-one.version=2.1.9 -Dhadoop-three.version=3.0.3 53 | ``` 54 | -------------------------------------------------------------------------------- /phoenix5-hive/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDateObjectInspectorBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.common.type.Date; 21 | import org.apache.hadoop.hive.serde2.io.DateWritableV2; 22 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; 23 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 24 | 25 | /** 26 | * ObjectInspector for date type 27 | */ 28 | 29 | public class PhoenixDateObjectInspectorBase extends AbstractPhoenixObjectInspector 30 | implements DateObjectInspector { 31 | 32 | public PhoenixDateObjectInspectorBase() { 33 | super(TypeInfoFactory.dateTypeInfo); 34 | } 35 | 36 | @Override 37 | public Object copyObject(Object o) { 38 | return o == null ? null : java.sql.Date.valueOf(o.toString()); 39 | } 40 | 41 | @Override 42 | public Date getPrimitiveJavaObject(Object o) { 43 | if (o == null) { 44 | return null; 45 | } 46 | return Date.valueOf(((java.sql.Date) o).toString()); 47 | } 48 | 49 | @Override 50 | public DateWritableV2 getPrimitiveWritableObject(Object o) { 51 | DateWritableV2 value = null; 52 | 53 | if (o != null) { 54 | try { 55 | value = new DateWritableV2(getPrimitiveJavaObject(o)); 56 | } catch (Exception e) { 57 | logExceptionMessage(o, "DATE"); 58 | value = new DateWritableV2(); 59 | } 60 | } 61 | 62 | return value; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /phoenix5-hive4/src/main/java/org/apache/phoenix/hive/objectinspector/PhoenixDateObjectInspectorBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.hive.objectinspector; 19 | 20 | import org.apache.hadoop.hive.common.type.Date; 21 | import org.apache.hadoop.hive.serde2.io.DateWritableV2; 22 | import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector; 23 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; 24 | 25 | /** 26 | * ObjectInspector for date type 27 | */ 28 | 29 | public class PhoenixDateObjectInspectorBase extends AbstractPhoenixObjectInspector 30 | implements DateObjectInspector { 31 | 32 | public PhoenixDateObjectInspectorBase() { 33 | super(TypeInfoFactory.dateTypeInfo); 34 | } 35 | 36 | @Override 37 | public Object copyObject(Object o) { 38 | return o == null ? null : java.sql.Date.valueOf(o.toString()); 39 | } 40 | 41 | @Override 42 | public Date getPrimitiveJavaObject(Object o) { 43 | if (o == null) { 44 | return null; 45 | } 46 | return Date.valueOf(((java.sql.Date) o).toString()); 47 | } 48 | 49 | @Override 50 | public DateWritableV2 getPrimitiveWritableObject(Object o) { 51 | DateWritableV2 value = null; 52 | 53 | if (o != null) { 54 | try { 55 | value = new DateWritableV2(getPrimitiveJavaObject(o)); 56 | } catch (Exception e) { 57 | logExceptionMessage(o, "DATE"); 58 | value = new DateWritableV2(); 59 | } 60 | } 61 | 62 | return value; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/java/org/apache/phoenix/spark/datasource/v2/reader/PhoenixDataSourceReadOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.phoenix.spark.datasource.v2.reader; 19 | 20 | import java.io.Serializable; 21 | import java.util.Properties; 22 | 23 | class PhoenixDataSourceReadOptions implements Serializable { 24 | 25 | private final String tenantId; 26 | private final String jdbcUrl; 27 | private final String scn; 28 | private final String selectStatement; 29 | private final Properties overriddenProps; 30 | private final byte[] pTableCacheBytes; 31 | 32 | PhoenixDataSourceReadOptions(String jdbcUrl, String scn, String tenantId, 33 | String selectStatement, Properties overriddenProps, byte[] pTableCacheBytes) { 34 | if (overriddenProps == null){ 35 | throw new NullPointerException(); 36 | } 37 | this.jdbcUrl = jdbcUrl; 38 | this.scn = scn; 39 | this.tenantId = tenantId; 40 | this.selectStatement = selectStatement; 41 | this.overriddenProps = overriddenProps; 42 | this.pTableCacheBytes = pTableCacheBytes; 43 | } 44 | 45 | String getSelectStatement() { 46 | return selectStatement; 47 | } 48 | 49 | String getScn() { 50 | return scn; 51 | } 52 | 53 | String getJdbcUrl() { 54 | return jdbcUrl; 55 | } 56 | 57 | String getTenantId() { 58 | return tenantId; 59 | } 60 | 61 | Properties getOverriddenProps() { 62 | return overriddenProps; 63 | } 64 | 65 | byte[] getPTableCacheBytes() { 66 | return pTableCacheBytes; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /phoenix5-spark/src/main/scala/org/apache/phoenix/spark/SparkContextFunctions.scala: -------------------------------------------------------------------------------- 1 | /* 2 | Licensed under the Apache License, Version 2.0 (the "License"); 3 | you may not use this file except in compliance with the License. 4 | You may obtain a copy of the License at 5 | 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | Unless required by applicable law or agreed to in writing, software 9 | distributed under the License is distributed on an "AS IS" BASIS, 10 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | See the License for the specific language governing permissions and 12 | limitations under the License. 13 | */ 14 | package org.apache.phoenix.spark 15 | 16 | import org.apache.hadoop.conf.Configuration 17 | import org.apache.spark.SparkContext 18 | import org.apache.spark.rdd.RDD 19 | import org.apache.spark.sql.SparkSession 20 | 21 | @deprecated("Use the DataSource V2 API implementation (see PhoenixDataSource)") 22 | class SparkContextFunctions(@transient val sc: SparkContext) extends Serializable { 23 | 24 | /* 25 | This will return an RDD of Map[String, AnyRef], where the String key corresponds to the column 26 | name and the AnyRef value will be a java.sql type as returned by Phoenix 27 | 28 | 'table' is the corresponding Phoenix table 29 | 'columns' is a sequence of of columns to query 30 | 'predicate' is a set of statements to go after a WHERE clause, e.g. "TID = 123" 31 | 'zkUrl' is an optional Zookeeper URL to use to connect to Phoenix 32 | 'conf' is a Hadoop Configuration object. If zkUrl is not set, the "hbase.zookeeper.quorum" 33 | property will be used 34 | */ 35 | 36 | def phoenixTableAsRDD(table: String, 37 | columns: Seq[String], 38 | predicate: Option[String] = None, 39 | zkUrl: Option[String] = None, 40 | tenantId: Option[String] = None, 41 | conf: Configuration = new Configuration()): RDD[Map[String, AnyRef]] = { 42 | 43 | val sparkSession = SparkSession.builder().config(sc.getConf).getOrCreate() 44 | val dataFrame = sparkSession.sqlContext.phoenixTableAsDataFrame( 45 | table = table, 46 | columns = columns, 47 | predicate = predicate, 48 | zkUrl = zkUrl, 49 | tenantId = tenantId, 50 | conf = conf 51 | ) 52 | 53 | dataFrame 54 | .rdd 55 | .map(row => row.getValuesMap(row.schema.fieldNames)) 56 | } 57 | } --------------------------------------------------------------------------------