├── conf ├── password-file.txt ├── wrong-password-file.txt ├── .gitignore ├── sqoop-env-template.sh └── sqoop-env-template.cmd ├── lib ├── ant-contrib-1.0b3.jar ├── ant-eclipse-1.0-jvm1.2.jar └── .gitignore ├── src ├── docs │ ├── web │ │ ├── images │ │ │ ├── tip.png │ │ │ ├── up.png │ │ │ ├── Thumbs.db │ │ │ ├── home.png │ │ │ ├── next.png │ │ │ ├── note.png │ │ │ ├── prev.png │ │ │ ├── caution.png │ │ │ ├── example.png │ │ │ ├── warning.png │ │ │ ├── callouts │ │ │ │ ├── 1.png │ │ │ │ ├── 10.png │ │ │ │ ├── 11.png │ │ │ │ ├── 12.png │ │ │ │ ├── 13.png │ │ │ │ ├── 14.png │ │ │ │ ├── 15.png │ │ │ │ ├── 2.png │ │ │ │ ├── 3.png │ │ │ │ ├── 4.png │ │ │ │ ├── 5.png │ │ │ │ ├── 6.png │ │ │ │ ├── 7.png │ │ │ │ ├── 8.png │ │ │ │ └── 9.png │ │ │ ├── important.png │ │ │ └── README │ │ ├── footer.xsl │ │ └── header.xsl │ ├── .gitignore │ ├── user │ │ ├── import-purpose.txt │ │ ├── import-mainframe-purpose.txt │ │ ├── input-formatting.txt │ │ ├── codegen-purpose.txt │ │ ├── job-purpose.txt │ │ ├── merge-purpose.txt │ │ ├── input-formatting-args.txt │ │ ├── eval-purpose.txt │ │ ├── hive-notes.txt │ │ ├── version.txt │ │ ├── import-all-tables-purpose.txt │ │ ├── export-purpose.txt │ │ ├── create-hive-table-purpose.txt │ │ ├── output-formatting-args.txt │ │ ├── support.txt │ │ ├── controlling-output-format.txt │ │ └── distributed-cache.txt │ ├── sip │ │ ├── INDEX.txt │ │ └── README.txt │ ├── man │ │ ├── sqoop-version.txt │ │ ├── common-args.txt │ │ ├── mainframe-connection-args.txt │ │ ├── sqoop-help.txt │ │ ├── sqoop-list-tables.txt │ │ ├── sqoop-metastore.txt │ │ ├── sqoop-eval.txt │ │ └── codegen-args.txt │ └── dev │ │ ├── intro.txt │ │ └── compiling.txt ├── test │ ├── org │ │ └── apache │ │ │ └── sqoop │ │ │ ├── testcategories │ │ │ ├── thirdpartytest │ │ │ │ ├── S3Test.java │ │ │ │ ├── Db2Test.java │ │ │ │ ├── MysqlTest.java │ │ │ │ ├── CubridTest.java │ │ │ │ ├── MainFrameTest.java │ │ │ │ ├── OracleTest.java │ │ │ │ ├── NetezzaTest.java │ │ │ │ ├── PostgresqlTest.java │ │ │ │ ├── SqlServerTest.java │ │ │ │ ├── ThirdPartyTest.java │ │ │ │ └── OracleEeTest.java │ │ │ ├── KerberizedTest.java │ │ │ └── sqooptest │ │ │ │ ├── ManualTest.java │ │ │ │ ├── UnitTest.java │ │ │ │ └── IntegrationTest.java │ │ │ ├── importjob │ │ │ ├── DatabaseAdapterFactory.java │ │ │ ├── configuration │ │ │ │ ├── AvroTestConfiguration.java │ │ │ │ ├── HiveTestConfiguration.java │ │ │ │ ├── ParquetTestConfiguration.java │ │ │ │ └── ImportJobTestConfiguration.java │ │ │ └── splitby │ │ │ │ ├── MysqlSplitByImportTest.java │ │ │ │ ├── OracleSplitByImportTest.java │ │ │ │ ├── PostgresSplitByImportTest.java │ │ │ │ └── SqlServerSplitByImportTest.java │ │ │ ├── infrastructure │ │ │ └── kerberos │ │ │ │ ├── MiniKdcInfrastructure.java │ │ │ │ └── KerberosConfigurationProvider.java │ │ │ ├── testutil │ │ │ ├── adapter │ │ │ │ └── DatabaseAdapter.java │ │ │ └── InjectableManagerFactory.java │ │ │ ├── manager │ │ │ ├── oracle │ │ │ │ ├── util │ │ │ │ │ ├── BinaryFloatGenerator.java │ │ │ │ │ ├── BinaryDoubleGenerator.java │ │ │ │ │ └── HadoopFiles.java │ │ │ │ └── TestOracleTable.java │ │ │ └── db2 │ │ │ │ └── DB2TestUtils.java │ │ │ ├── cloud │ │ │ └── s3 │ │ │ │ ├── TestS3TextImport.java │ │ │ │ ├── TestS3AvroImport.java │ │ │ │ ├── TestS3ParquetImport.java │ │ │ │ ├── TestS3SequenceFileImport.java │ │ │ │ ├── TestS3IncrementalMergeTextImport.java │ │ │ │ ├── TestS3IncrementalAppendAvroImport.java │ │ │ │ ├── TestS3IncrementalAppendTextImport.java │ │ │ │ ├── TestS3IncrementalMergeParquetImport.java │ │ │ │ ├── TestS3IncrementalAppendParquetImport.java │ │ │ │ └── TestS3IncrementalAppendSequenceFileImport.java │ │ │ ├── hive │ │ │ └── minicluster │ │ │ │ └── AuthenticationConfiguration.java │ │ │ ├── hbase │ │ │ └── HBaseImportTypesTest.java │ │ │ └── metastore │ │ │ └── hsqldb │ │ │ └── HsqldbJobToolTest.java │ ├── checkstyle-java-header.txt │ ├── fi-site.xml │ └── oraoop │ │ └── pkg_tst_product_gen.psk ├── scripts │ ├── thirdpartytest │ │ ├── docker-compose │ │ │ └── oraclescripts │ │ │ │ ├── healthcheck.sh │ │ │ │ ├── ee-healthcheck.sh │ │ │ │ └── startup │ │ │ │ └── oracleusersetup.sql │ │ ├── start-thirdpartytest-db-containers.sh │ │ └── stop-thirdpartytest-db-containers.sh │ ├── tool-script.cmd.template │ └── run-perftest.sh ├── java │ └── org │ │ └── apache │ │ └── sqoop │ │ ├── db │ │ ├── JdbcConnectionFactory.java │ │ └── decorator │ │ │ └── JdbcConnectionFactoryDecorator.java │ │ ├── hive │ │ └── HiveClient.java │ │ ├── mapreduce │ │ ├── db │ │ │ ├── NTextSplitter.java │ │ │ ├── netezza │ │ │ │ ├── NetezzaExternalTableTextExportMapper.java │ │ │ │ └── NetezzaExternalTableRecordExportMapper.java │ │ │ └── OracleDateSplitter.java │ │ ├── MergeRecordMapper.java │ │ ├── parquet │ │ │ ├── ParquetConstants.java │ │ │ ├── hadoop │ │ │ │ ├── HadoopMergeParquetReducer.java │ │ │ │ └── HadoopParquetExportMapper.java │ │ │ ├── ParquetMergeJobConfigurator.java │ │ │ └── ParquetJobConfiguratorFactory.java │ │ ├── mainframe │ │ │ ├── MainframeDatasetImportMapper.java │ │ │ └── MainframeDatasetType.java │ │ ├── MergeReducer.java │ │ ├── HBaseImportMapper.java │ │ └── AccumuloImportMapper.java │ │ ├── validation │ │ ├── ValidationThreshold.java │ │ ├── ValidationException.java │ │ └── ValidationFailureHandler.java │ │ ├── util │ │ ├── ErrorableThread.java │ │ ├── ErrorableAsyncSink.java │ │ ├── StoredAsProperty.java │ │ ├── ExportException.java │ │ └── ImportException.java │ │ ├── lib │ │ ├── FieldMappable.java │ │ └── FieldMapProcessor.java │ │ ├── io │ │ └── UnsupportedCodecException.java │ │ ├── tool │ │ ├── ToolPlugin.java │ │ ├── VersionTool.java │ │ └── JDBCTransactionLevels.java │ │ ├── manager │ │ └── oracle │ │ │ ├── OracleActiveInstance.java │ │ │ ├── OraOopOracleDataChunk.java │ │ │ └── OracleTableColumns.java │ │ └── cli │ │ └── RelatedOptions.java └── perftest │ └── README-perftest.txt ├── gradle ├── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties └── customWindowsStartScript.txt ├── NOTICE.txt ├── README.md ├── bin ├── .gitignore └── sqoop.cmd ├── testdata ├── DatatypeTestData-export-lite.txt ├── DatatypeTestData-import-lite.txt └── hive │ └── scripts │ ├── createOnlyImport.q │ ├── createOverwriteImport.q │ ├── dateImport.q │ ├── failingImport.q │ ├── numericImport.q │ ├── normalImport.q │ ├── customDelimImport.q │ ├── decimalMapImport.q │ ├── incrementalHiveAppend10.q │ ├── incrementalHiveAppend20.q │ ├── incrementalHiveAppendEmpty.q │ ├── fieldWithNewlineImport.q │ ├── partitionImport.q │ └── fieldWithNewlineReplacementImport.q ├── config └── checkstyle │ └── checkstyle-java-header.txt ├── settings.gradle ├── .gitignore └── .gitattributes /conf/password-file.txt: -------------------------------------------------------------------------------- 1 | credProviderPwd -------------------------------------------------------------------------------- /conf/wrong-password-file.txt: -------------------------------------------------------------------------------- 1 | wrongCredProviderPwd -------------------------------------------------------------------------------- /lib/ant-contrib-1.0b3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/lib/ant-contrib-1.0b3.jar -------------------------------------------------------------------------------- /src/docs/web/images/tip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/tip.png -------------------------------------------------------------------------------- /src/docs/web/images/up.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/up.png -------------------------------------------------------------------------------- /lib/ant-eclipse-1.0-jvm1.2.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/lib/ant-eclipse-1.0-jvm1.2.jar -------------------------------------------------------------------------------- /src/docs/web/images/Thumbs.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/Thumbs.db -------------------------------------------------------------------------------- /src/docs/web/images/home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/home.png -------------------------------------------------------------------------------- /src/docs/web/images/next.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/next.png -------------------------------------------------------------------------------- /src/docs/web/images/note.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/note.png -------------------------------------------------------------------------------- /src/docs/web/images/prev.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/prev.png -------------------------------------------------------------------------------- /src/docs/web/images/caution.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/caution.png -------------------------------------------------------------------------------- /src/docs/web/images/example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/example.png -------------------------------------------------------------------------------- /src/docs/web/images/warning.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/warning.png -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /src/docs/web/images/callouts/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/1.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/10.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/11.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/12.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/13.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/14.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/15.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/2.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/3.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/4.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/5.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/6.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/7.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/8.png -------------------------------------------------------------------------------- /src/docs/web/images/callouts/9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/callouts/9.png -------------------------------------------------------------------------------- /src/docs/web/images/important.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/sqoop/HEAD/src/docs/web/images/important.png -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /src/docs/web/images/README: -------------------------------------------------------------------------------- 1 | Replaced the plain DocBook XSL admonition icons with Jimmac's DocBook 2 | icons (http://jimmac.musichall.cz/ikony.php3). I dropped transparency 3 | from the Jimmac icons to get round MS IE and FOP PNG incompatibilies. 4 | 5 | Stuart Rackham 6 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/S3Test.java: -------------------------------------------------------------------------------- 1 | package org.apache.sqoop.testcategories.thirdpartytest; 2 | 3 | /** 4 | * An S3 test shall test the integration with the Amazon S3 cloud service. 5 | * These tests also require AWS credentials to access S3 and they run only if these 6 | * credentials are provided via the -Ds3.generator.command= property 7 | * as well as the target S3 location via the -Ds3.bucket.url= property. 8 | */ 9 | public interface S3Test extends ThirdPartyTest { 10 | } 11 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Apache Sqoop 2 | Copyright 2011-2013 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | Portions of this software were developed at 8 | Cloudera, Inc. (http://www.cloudera.com/). 9 | 10 | Patch review tool is based on the Kafka patch review tool. 11 | Which is part of the Apache Kafka project, an open source software 12 | project with copyright by The Apache Software Foundation. 13 | The original source code is available at: http://kafka.apache.org 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/apache/sqoop.svg?branch=trunk)](https://travis-ci.org/apache/sqoop) 2 | 3 | # Welcome to Sqoop! 4 | 5 | This is the Sqoop (SQL-to-Hadoop) tool. Sqoop allows easy imports and 6 | exports of data sets between databases and HDFS. 7 | 8 | ## More Documentation 9 | 10 | Sqoop ships with additional documentation: a user guide and a manual page. 11 | 12 | Asciidoc sources for both of these are in `src/docs/`. Run `ant docs` to build 13 | the documentation. It will be created in `build/docs/`. 14 | 15 | If you got Sqoop in release form, documentation will already be built and 16 | available in the `docs/` directory. 17 | 18 | For more information on compiling see [COMPILING.adoc](COMPILING.adoc). 19 | -------------------------------------------------------------------------------- /lib/.gitignore: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | /ivy-*.jar 16 | -------------------------------------------------------------------------------- /bin/.gitignore: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | /sqoop-* 16 | 17 | -------------------------------------------------------------------------------- /conf/.gitignore: -------------------------------------------------------------------------------- 1 | # Licensed to Cloudera, Inc. under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # Cloudera, Inc. licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | /sqoop-site.xml 16 | /managers.d 17 | -------------------------------------------------------------------------------- /src/test/checkstyle-java-header.txt: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | -------------------------------------------------------------------------------- /testdata/DatatypeTestData-export-lite.txt: -------------------------------------------------------------------------------- 1 | BIGINT,9223372036854775807,9223372036854775807,9223372036854775807,POS 2 | DECIMAL,999999999999999999,999999999999999999,999999999999999999,18,0,POS 3 | INT,2147483647,2147483647,2147483647,POS 4 | MONEY,9223372036,9223372036.0000,9223372036.0000,POS 5 | SMALLINT,-32768,-32768,-32768,POS 6 | TINYINT,0,0,0,POS 7 | FLOAT,-2.23E-308,-2.23E-308,-2.23E-308,POS 8 | REAL,1000.0001,1000.0001,1000.0001,POS 9 | DATE,0001-01-01,0001-01-01,0001-01-01,POS 10 | TIME,00:00:00.0000000,00:00:00.0000000,00:00:00,POS 11 | CHAR,P,P,P,1,POS 12 | VARCHAR,P,P,P,1,POS 13 | NCHAR,L,L,L,1,POS 14 | NVARCHAR,P,P,P,1,POS 15 | SMALLDATETIME, 1900-01-01 23:58:58,1900-01-01 23:59:00.0,1900-01-01 23:59:00.0,POS 16 | DATETIME2,9999-12-31 23:59:59.90,9999-12-31 23:59:59.9000000,9999-12-31 23:59:59.9,POS 17 | DATETIME,9999-12-31 23:59:59.997,9999-12-31 23:59:59.997,9999-12-31 23:59:59.997,POS 18 | -------------------------------------------------------------------------------- /config/checkstyle/checkstyle-java-header.txt: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | rootProject.name ='Sqoop' 19 | enableFeaturePreview('STABLE_PUBLISHING') 20 | -------------------------------------------------------------------------------- /src/docs/.gitignore: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | /Sqoop-manpage.xml 18 | /sqoop.1 19 | /Sqoop-web.html 20 | /dev/SqoopDevGuide.xml 21 | /user/SqoopUserGuide.xml 22 | -------------------------------------------------------------------------------- /testdata/DatatypeTestData-import-lite.txt: -------------------------------------------------------------------------------- 1 | BIGINT,-9223372036854775808,-9223372036854775808,-9223372036854775808,POS 2 | DECIMAL,1000.0001,1000.00010,1000.00010,10,5,POS 3 | INT,-2147483648,-2147483648,-2147483648,POS 4 | MONEY,92233720,92233720.0000,92233720.0000,POS 5 | SMALLINT,-32768,-32768,-32768,POS 6 | SMALLMONEY,-214748.3648,-214748.3648,-214748.3648,POS 7 | TINYINT,0,0,0,POS 8 | FLOAT,1.797693134862,1.797693134862,1.797693134862,POS 9 | REAL,3.4028234663852886,3.4028234,3.4028234,POS 10 | DATE,0001-01-01,0001-01-01,0001-01-01,POS 11 | DATETIME2,0001-01-01 00:00:00,0001-01-01 00:00:00.0000000,0001-01-01 00:00:00.0,POS 12 | SMALLDATETIME,1900-01-01,1900-01-01 00:00:00.0,1900-01-01 00:00:00.0,POS 13 | DATETIME,1753-01-01 00:00:00,1753-01-01 00:00:00.0,1753-01-01 00:00:00.0,POS 14 | TIME,00:00:00.0000000,00:00:00.0000000,00:00:00,POS 15 | CHAR,P,P,P,1,POS 16 | VARCHAR,P,P,P,1,POS 17 | NCHAR,L,L,L,1,POS 18 | NVARCHAR,P,P,P,1,POS 19 | BINARY,1,31,31,1,POS 20 | VARBINARY,1,31,31,100,POS -------------------------------------------------------------------------------- /src/scripts/thirdpartytest/docker-compose/oraclescripts/healthcheck.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | echo "SELECT 1 from DUAL;" | sqlplus -L SQOOPTEST2/ABCDEF@//localhost:1521/xe 20 | -------------------------------------------------------------------------------- /src/scripts/thirdpartytest/docker-compose/oraclescripts/ee-healthcheck.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | 19 | echo "SELECT 1 from DUAL;" | sqlplus -L SQOOPTEST2/ABCDEF@//localhost:1521/sqoop 20 | -------------------------------------------------------------------------------- /src/scripts/thirdpartytest/start-thirdpartytest-db-containers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | BASEDIR=$(dirname "$0") 19 | 20 | docker-compose -f $BASEDIR/docker-compose/sqoop-thirdpartytest-db-services.yml up -d "$@" 21 | -------------------------------------------------------------------------------- /src/scripts/thirdpartytest/stop-thirdpartytest-db-containers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | 18 | BASEDIR=$(dirname "$0") 19 | 20 | docker-compose -f $BASEDIR/docker-compose/sqoop-thirdpartytest-db-services.yml rm --stop -f 21 | -------------------------------------------------------------------------------- /gradle/customWindowsStartScript.txt: -------------------------------------------------------------------------------- 1 | @echo off 2 | :: Licensed to the Apache Software Foundation (ASF) under one or more 3 | :: contributor license agreements. See the NOTICE file distributed with 4 | :: this work for additional information regarding copyright ownership. 5 | :: The ASF licenses this file to You under the Apache License, Version 2.0 6 | :: (the "License"); you may not use this file except in compliance with 7 | :: the License. You may obtain a copy of the License at 8 | :: 9 | :: 10 | :: http://www.apache.org/licenses/LICENSE-2.0 11 | :: 12 | :: Unless required by applicable law or agreed to in writing, software 13 | :: distributed under the License is distributed on an "AS IS" BASIS, 14 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | :: See the License for the specific language governing permissions and 16 | :: limitations under the License. 17 | setlocal 18 | 19 | set prgm=%~f0 20 | set bin=%~dp0 21 | 22 | if "%bin:~-1%" == "\" ( 23 | set bin=%bin:~0,-1% 24 | ) 25 | 26 | call "%bin%\\sqoop.cmd" SUBCOMMANDMARKER %* 27 | -------------------------------------------------------------------------------- /src/scripts/tool-script.cmd.template: -------------------------------------------------------------------------------- 1 | @echo off 2 | :: Licensed to the Apache Software Foundation (ASF) under one or more 3 | :: contributor license agreements. See the NOTICE file distributed with 4 | :: this work for additional information regarding copyright ownership. 5 | :: The ASF licenses this file to You under the Apache License, Version 2.0 6 | :: (the "License"); you may not use this file except in compliance with 7 | :: the License. You may obtain a copy of the License at 8 | :: 9 | :: 10 | :: http://www.apache.org/licenses/LICENSE-2.0 11 | :: 12 | :: Unless required by applicable law or agreed to in writing, software 13 | :: distributed under the License is distributed on an "AS IS" BASIS, 14 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | :: See the License for the specific language governing permissions and 16 | :: limitations under the License. 17 | 18 | setlocal 19 | 20 | set prgm=%~f0 21 | set bin=%~dp0 22 | if "%bin:~-1%" == "\" ( 23 | set bin=%bin:~0,-1% 24 | ) 25 | 26 | call "%bin%\sqoop.cmd" SUBCOMMANDMARKER %* 27 | -------------------------------------------------------------------------------- /testdata/hive/scripts/createOnlyImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `CREATE_ONLY_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | -------------------------------------------------------------------------------- /testdata/hive/scripts/createOverwriteImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE `CREATE_OVERWRITE_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/db/JdbcConnectionFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.db; 20 | 21 | import java.sql.Connection; 22 | 23 | public interface JdbcConnectionFactory { 24 | 25 | Connection createConnection(); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/scripts/thirdpartytest/docker-compose/oraclescripts/startup/oracleusersetup.sql: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one or more 2 | -- contributor license agreements. See the NOTICE file distributed with 3 | -- this work for additional information regarding copyright ownership. 4 | -- The ASF licenses this file to You under the Apache License, Version 2.0 5 | -- (the "License"); you may not use this file except in compliance with 6 | -- the License. You may obtain a copy of the License at 7 | -- 8 | -- http://www.apache.org/licenses/LICENSE-2.0 9 | -- 10 | -- Unless required by applicable law or agreed to in writing, software 11 | -- distributed under the License is distributed on an "AS IS" BASIS, 12 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | -- See the License for the specific language governing permissions and 14 | -- limitations under the License. 15 | 16 | ALTER SESSION SET "_ORACLE_SCRIPT"=true; 17 | CREATE USER SQOOPTEST2 identified by ABCDEF; 18 | GRANT CONNECT, RESOURCE, DBA to SQOOPTEST2; 19 | ALTER USER SQOOPTEST2 quota unlimited on USERS; 20 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/KerberizedTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories; 19 | 20 | /** 21 | * A kerberized test shall run in kerberized environment thus it starts mini KDC server. 22 | */ 23 | public interface KerberizedTest { 24 | } 25 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/sqooptest/ManualTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.sqooptest; 19 | 20 | /** 21 | * Deprecated category, shall not be used nor extended. 22 | */ 23 | public interface ManualTest extends IntegrationTest { 24 | } 25 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/hive/HiveClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.hive; 20 | 21 | import java.io.IOException; 22 | 23 | public interface HiveClient { 24 | 25 | void importTable() throws IOException; 26 | 27 | void createTable() throws IOException; 28 | } 29 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/DatabaseAdapterFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob; 20 | 21 | import org.apache.sqoop.testutil.adapter.DatabaseAdapter; 22 | 23 | public interface DatabaseAdapterFactory { 24 | 25 | DatabaseAdapter createAdapter(); 26 | } 27 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/configuration/AvroTestConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob.configuration; 20 | 21 | public interface AvroTestConfiguration extends ImportJobTestConfiguration { 22 | 23 | String[] getExpectedResultsForAvro(); 24 | } 25 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/configuration/HiveTestConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob.configuration; 20 | 21 | public interface HiveTestConfiguration extends ImportJobTestConfiguration{ 22 | 23 | Object[] getExpectedResultsForHive(); 24 | } 25 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/configuration/ParquetTestConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob.configuration; 20 | 21 | public interface ParquetTestConfiguration extends ImportJobTestConfiguration{ 22 | 23 | String[] getExpectedResultsForParquet(); 24 | } 25 | -------------------------------------------------------------------------------- /testdata/hive/scripts/dateImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `DATE_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/DATE_HIVE_IMPORT' INTO TABLE `DATE_HIVE_IMPORT`; 18 | -------------------------------------------------------------------------------- /testdata/hive/scripts/failingImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `DATE_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/DATE_HIVE_IMPORT' INTO TABLE `DATE_HIVE_IMPORT`; 18 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/infrastructure/kerberos/MiniKdcInfrastructure.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.infrastructure.kerberos; 20 | 21 | public interface MiniKdcInfrastructure extends KerberosConfigurationProvider { 22 | 23 | void start(); 24 | 25 | void stop(); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /testdata/hive/scripts/numericImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `NUMERIC_HIVE_IMPORT` ( `DATA_COL0` DOUBLE, `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/NUMERIC_HIVE_IMPORT' INTO TABLE `NUMERIC_HIVE_IMPORT`; 18 | -------------------------------------------------------------------------------- /src/docs/user/import-purpose.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | The +import+ tool imports an individual table from an RDBMS to HDFS. 21 | Each row from a table is represented as a separate record in HDFS. 22 | Records can be stored as text files (one record per line), or in 23 | binary representation as Avro or SequenceFiles. 24 | 25 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/Db2Test.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * A DB2 test shall test scenarios where a DB2 driver and/or external instance is required. 22 | */ 23 | public interface Db2Test extends ThirdPartyTest { 24 | } 25 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/MysqlTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * A MySql test shall test scenarios where a MySql driver and/or external instance is required. 22 | */ 23 | public interface MysqlTest extends ThirdPartyTest { 24 | } 25 | -------------------------------------------------------------------------------- /testdata/hive/scripts/normalImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `NORMAL_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/NORMAL_HIVE_IMPORT' INTO TABLE `NORMAL_HIVE_IMPORT`; 18 | -------------------------------------------------------------------------------- /bin/sqoop.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | :: Licensed to the Apache Software Foundation (ASF) under one or more 3 | :: contributor license agreements. See the NOTICE file distributed with 4 | :: this work for additional information regarding copyright ownership. 5 | :: The ASF licenses this file to You under the Apache License, Version 2.0 6 | :: (the "License"); you may not use this file except in compliance with 7 | :: the License. You may obtain a copy of the License at 8 | :: 9 | :: 10 | :: http://www.apache.org/licenses/LICENSE-2.0 11 | :: 12 | :: Unless required by applicable law or agreed to in writing, software 13 | :: distributed under the License is distributed on an "AS IS" BASIS, 14 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | :: See the License for the specific language governing permissions and 16 | :: limitations under the License. 17 | 18 | setlocal enabledelayedexpansion 19 | 20 | set prgm=%~f0 21 | set bin=%~dp0 22 | if "%bin:~-1%" == "\" ( 23 | set bin=%bin:~0,-1% 24 | ) 25 | 26 | call "%bin%\configure-sqoop.cmd" "%bin%" 27 | call %HADOOP_HOME%\bin\hadoop org.apache.sqoop.Sqoop %* 28 | 29 | endlocal 30 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/db/NTextSplitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.mapreduce.db; 19 | 20 | /** 21 | * Implement DBSplitter over native text strings. 22 | */ 23 | public class NTextSplitter extends TextSplitter { 24 | 25 | public NTextSplitter() { 26 | setUseNCharStrings(true); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/CubridTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * A CubridTest shall test scenarios where a Cubrid driver and/or external instance is required. 22 | */ 23 | public interface CubridTest extends ThirdPartyTest { 24 | } 25 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/MainFrameTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * A MainFrame test shall test scenarios where a MainFrame external instance is required. 22 | */ 23 | public interface MainFrameTest extends ThirdPartyTest { 24 | } 25 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/OracleTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * An Oracle test shall test scenarios where a Oracle driver and/or external instance is required. 22 | */ 23 | public interface OracleTest extends ThirdPartyTest { 24 | } 25 | -------------------------------------------------------------------------------- /testdata/hive/scripts/customDelimImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `CUSTOM_DELIM_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\054' LINES TERMINATED BY '\174' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/CUSTOM_DELIM_IMPORT' INTO TABLE `CUSTOM_DELIM_IMPORT`; 18 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/NetezzaTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * A Netezza test shall test scenarios where a Netezza driver and/or external instance is required. 22 | */ 23 | public interface NetezzaTest extends ThirdPartyTest { 24 | } 25 | -------------------------------------------------------------------------------- /testdata/hive/scripts/decimalMapImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `DECIMAL_MAP_HIVE_IMPORT` ( `DATA_COL0` DECIMAL(10, 10), `DATA_COL1` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/DECIMAL_MAP_HIVE_IMPORT' INTO TABLE `DECIMAL_MAP_HIVE_IMPORT`; 18 | -------------------------------------------------------------------------------- /testdata/hive/scripts/incrementalHiveAppend10.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; -------------------------------------------------------------------------------- /testdata/hive/scripts/incrementalHiveAppend20.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; -------------------------------------------------------------------------------- /src/docs/sip/INDEX.txt: -------------------------------------------------------------------------------- 1 | 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | 18 | This is the index of all accepted SIPs: 19 | 20 | * SIP-1 – Providing multiple entry-points into Sqoop 21 | * SIP-2 – Sqoop 1.0 release criteria and maintenence policy 22 | * SIP-3 – File format for large object (LOB) storage 23 | * SIP-4 – Public API for Sqoop v1.0.0 24 | * SIP-6 – Reusable Jobs for Sqoop Operations 25 | 26 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/infrastructure/kerberos/KerberosConfigurationProvider.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.infrastructure.kerberos; 20 | 21 | public interface KerberosConfigurationProvider { 22 | 23 | String getTestPrincipal(); 24 | 25 | String getRealm(); 26 | 27 | String getKeytabFilePath(); 28 | 29 | } 30 | -------------------------------------------------------------------------------- /testdata/hive/scripts/incrementalHiveAppendEmpty.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`; -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/PostgresqlTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * A PostgreSql test shall test scenarios where a PostgreSql driver and/or external instance is required. 22 | */ 23 | public interface PostgresqlTest extends ThirdPartyTest { 24 | } 25 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/SqlServerTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * An SqlServer test shall test scenarios where a SqlServer driver and/or external instance is required. 22 | */ 23 | public interface SqlServerTest extends ThirdPartyTest { 24 | } 25 | -------------------------------------------------------------------------------- /testdata/hive/scripts/fieldWithNewlineImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `FIELD_WITH_NL_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/FIELD_WITH_NL_HIVE_IMPORT' INTO TABLE `FIELD_WITH_NL_HIVE_IMPORT`; 18 | -------------------------------------------------------------------------------- /src/docs/user/import-mainframe-purpose.txt: -------------------------------------------------------------------------------- 1 | //// 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | //// 18 | 19 | The +import-mainframe+ tool imports all sequential datasets 20 | in a partitioned dataset(PDS) on a mainframe to HDFS. A PDS is 21 | akin to a directory on the open systems. 22 | The records in a dataset can contain only character data. 23 | Records will be stored with the entire record as a single text field. 24 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/configuration/ImportJobTestConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob.configuration; 20 | 21 | import java.util.List; 22 | 23 | public interface ImportJobTestConfiguration { 24 | 25 | String[] getTypes(); 26 | 27 | String[] getNames(); 28 | 29 | List getSampleData(); 30 | } 31 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/ThirdPartyTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.thirdpartytest; 19 | 20 | /** 21 | * A third party test shall test a scenario where a third party side is required such as a JDBC driver or an external 22 | * RDBMS instance. 23 | */ 24 | public interface ThirdPartyTest { 25 | } 26 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/thirdpartytest/OracleEeTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.testcategories.thirdpartytest; 20 | 21 | /** 22 | * An Oracle EE test shall test scenarios where a Oracle Enterprise Edition database is needed, 23 | * for example because of partitioning. 24 | */ 25 | public interface OracleEeTest extends OracleTest { 26 | } 27 | -------------------------------------------------------------------------------- /testdata/hive/scripts/partitionImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `PARTITION_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) PARTITIONED BY (ds STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/PARTITION_HIVE_IMPORT' INTO TABLE `PARTITION_HIVE_IMPORT` PARTITION (ds='20110413'); 18 | -------------------------------------------------------------------------------- /src/docs/man/sqoop-version.txt: -------------------------------------------------------------------------------- 1 | sqoop-version(1) 2 | ================ 3 | 4 | NAME 5 | ---- 6 | sqoop-version - Display version information for Sqoop. 7 | 8 | SYNOPSIS 9 | -------- 10 | 'sqoop-version' 11 | 12 | 'sqoop version' 13 | 14 | 15 | DESCRIPTION 16 | ----------- 17 | 18 | Display version information. 19 | 20 | //// 21 | Copyright 2011 The Apache Software Foundation 22 | 23 | Licensed to the Apache Software Foundation (ASF) under one 24 | or more contributor license agreements. See the NOTICE file 25 | distributed with this work for additional information 26 | regarding copyright ownership. The ASF licenses this file 27 | to you under the Apache License, Version 2.0 (the 28 | "License"); you may not use this file except in compliance 29 | with the License. You may obtain a copy of the License at 30 | 31 | http://www.apache.org/licenses/LICENSE-2.0 32 | 33 | Unless required by applicable law or agreed to in writing, software 34 | distributed under the License is distributed on an "AS IS" BASIS, 35 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 36 | See the License for the specific language governing permissions and 37 | limitations under the License. 38 | //// 39 | 40 | -------------------------------------------------------------------------------- /testdata/hive/scripts/fieldWithNewlineReplacementImport.q: -------------------------------------------------------------------------------- 1 | -- Licensed to the Apache Software Foundation (ASF) under one 2 | -- or more contributor license agreements. See the NOTICE file 3 | -- distributed with this work for additional information 4 | -- regarding copyright ownership. The ASF licenses this file 5 | -- to you under the Apache License, Version 2.0 (the 6 | -- "License"); you may not use this file except in compliance 7 | -- with the License. You may obtain a copy of the License at 8 | -- 9 | -- http://www.apache.org/licenses/LICENSE-2.0 10 | -- 11 | -- Unless required by applicable law or agreed to in writing, software 12 | -- distributed under the License is distributed on an "AS IS" BASIS, 13 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | -- See the License for the specific language governing permissions and 15 | -- limitations under the License. 16 | CREATE TABLE IF NOT EXISTS `FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT` ( `DATA_COL0` STRING, `DATA_COL1` INT, `DATA_COL2` STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE; 17 | LOAD DATA INPATH 'file:BASEPATH/FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT' INTO TABLE `FIELD_WITH_NL_REPLACEMENT_HIVE_IMPORT`; 18 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/sqooptest/UnitTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.sqooptest; 19 | 20 | /** 21 | * A unit test shall test one class at a time having it's dependencies mocked. 22 | * A unit test shall not start a mini cluster nor an embedded database and it shall not use a JDBC driver. 23 | */ 24 | public interface UnitTest extends SqoopTest { 25 | } 26 | -------------------------------------------------------------------------------- /src/docs/user/input-formatting.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | Record classes generated by Sqoop include both a +toString()+ method 21 | that formats output records, and a +parse()+ method that interprets 22 | text based on an input delimiter set. The input delimiters default to 23 | the same ones chosen for output delimiters, but you can override these 24 | settings to support converting from one set of delimiters to another. 25 | 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | *.eml 18 | *.iml 19 | *.ipr 20 | *.iws 21 | *.log 22 | *~ 23 | .classpath 24 | .idea 25 | .project 26 | .settings 27 | nb-configuration.xml 28 | patch-process 29 | target 30 | 31 | # Sqoop1 specific 32 | .ant-targets-build.xml 33 | .launches 34 | /build 35 | /build.properties 36 | /conf/managers.d 37 | /conf/tools.d 38 | /tags 39 | src/java/org/apache/sqoop/SqoopVersion.java 40 | gradle/build 41 | gradle/.gradle 42 | .gradle 43 | out 44 | -------------------------------------------------------------------------------- /src/docs/dev/intro.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | 21 | Introduction 22 | ------------ 23 | 24 | If you are a developer or an application programmer who intends to 25 | modify Sqoop or build an extension using one of Sqoop's internal APIs, 26 | you should read this document. The following sections describe the 27 | purpose of each API, where internal APIs are used, and which APIs are 28 | necessary for implementing support for additional databases. 29 | 30 | -------------------------------------------------------------------------------- /src/docs/man/common-args.txt: -------------------------------------------------------------------------------- 1 | //// 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | //// 18 | 19 | Database connection and common options 20 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 21 | 22 | --connect (jdbc-uri):: 23 | Specify JDBC connect string (required) 24 | 25 | --driver (class-name):: 26 | Manually specify JDBC driver class to use 27 | 28 | --connection-param-file (filename):: 29 | Optional properties file that provides connection parameters 30 | 31 | include::database-independent-args.txt[] 32 | -------------------------------------------------------------------------------- /src/docs/user/codegen-purpose.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | The +codegen+ tool generates Java classes which encapsulate and 21 | interpret imported records. The Java definition of a record is 22 | instantiated as part of the import process, but can also be performed 23 | separately. For example, if Java source is lost, it can be recreated. 24 | New versions of a class can be created which use different delimiters 25 | between fields, and so on. 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/validation/ValidationThreshold.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.validation; 20 | 21 | /** 22 | * This is the primary interface that is responsible for driving the actual 23 | * decision on validation based on an optional error margin threshold. 24 | */ 25 | public interface ValidationThreshold { 26 | 27 | void setThresholdValue(long value); 28 | 29 | boolean compare(Comparable left, Comparable right); 30 | } 31 | -------------------------------------------------------------------------------- /src/docs/user/job-purpose.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | The job tool allows you to create and work with saved jobs. Saved jobs 21 | remember the parameters used to specify a job, so they can be 22 | re-executed by invoking the job by its handle. 23 | 24 | If a saved job is configured to perform an incremental import, state regarding 25 | the most recently imported rows is updated in the saved job to allow the job 26 | to continually import only the newest rows. 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/docs/user/merge-purpose.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | The merge tool allows you to combine two datasets where entries in one 21 | dataset should overwrite entries of an older dataset. For example, an 22 | incremental import run in last-modified mode will generate multiple datasets 23 | in HDFS where successively newer data appears in each dataset. The +merge+ 24 | tool will "flatten" two datasets into one, taking the newest available 25 | records for each primary key. 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/docs/user/input-formatting-args.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | 21 | --input-fields-terminated-by (char):: 22 | Sets the input field separator 23 | 24 | --input-lines-terminated-by (char):: 25 | Sets the input end-of-line char 26 | 27 | --input-optionally-enclosed-by (char):: 28 | Sets an input field-enclosing character 29 | 30 | --input-enclosed-by (char):: 31 | Sets a required input field encloser 32 | 33 | --input-escaped-by (char):: 34 | Sets the input escape character 35 | 36 | -------------------------------------------------------------------------------- /src/test/fi-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | fi.* 25 | 0.10 26 | 27 | Default probability level for all injected faults specified 28 | as a floating number between 0 and 1.00 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/docs/user/eval-purpose.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | The +eval+ tool allows users to quickly run simple SQL queries against 21 | a database; results are printed to the console. This allows users to 22 | preview their import queries to ensure they import the data they 23 | expect. 24 | 25 | WARNING: The +eval+ tool is provided for evaluation purpose only. You can use it to verify database connection from within the Sqoop or to test simple queries. It's not suppose to be used in production workflows. 26 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testcategories/sqooptest/IntegrationTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.testcategories.sqooptest; 19 | 20 | /** 21 | * An integration test shall test if independently developed classes work together correctly. 22 | * An integration test checks a whole scenario and thus may start mini clusters or embedded databases and may connect to 23 | * external resources like RDBMS instances. 24 | */ 25 | public interface IntegrationTest extends SqoopTest { 26 | } 27 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # Auto detect text files and perform LF normalization 19 | * text=auto 20 | 21 | *.cs text diff=csharp 22 | *.java text diff=java 23 | *.html text diff=html 24 | *.py text diff=python 25 | *.pl text diff=perl 26 | *.pm text diff=perl 27 | *.css text 28 | *.js text 29 | *.sql text 30 | 31 | *.sh text eol=lf 32 | 33 | *.bat text eol=crlf 34 | *.cmd text eol=crlf 35 | *.csproj text merge=union eol=crlf 36 | *.sln text merge=union eol=crlf 37 | -------------------------------------------------------------------------------- /src/docs/man/mainframe-connection-args.txt: -------------------------------------------------------------------------------- 1 | //// 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | //// 18 | 19 | .Connection arguments 20 | 21 | --connect (hostname):: 22 | Specify mainframe host name (required) 23 | 24 | --dataset (partitioned dataset name):: 25 | Specify a partitioned dataset name 26 | 27 | --datasettype (data set type):: 28 | Specify a dataset type. "s" for sequential, "p" for partitioned dataset (default), "g" for generational data group 29 | 30 | --tape (dataset on tape):: 31 | Specify whether a dataset is on tape or not (true|false) -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testutil/adapter/DatabaseAdapter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.testutil.adapter; 20 | 21 | import org.apache.sqoop.SqoopOptions; 22 | import org.apache.sqoop.manager.ConnManager; 23 | 24 | import java.sql.SQLException; 25 | 26 | public interface DatabaseAdapter { 27 | 28 | String getConnectionString(); 29 | 30 | SqoopOptions injectConnectionParameters(SqoopOptions options); 31 | 32 | void dropTableIfExists(String tableName, ConnManager manager) throws SQLException; 33 | } 34 | -------------------------------------------------------------------------------- /src/docs/user/hive-notes.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | 21 | 22 | Schema Definition in Hive 23 | ~~~~~~~~~~~~~~~~~~~~~~~~~ 24 | 25 | Hive users will note that there is not a one-to-one mapping between 26 | SQL types and Hive types. In general, SQL types that do not have a 27 | direct mapping (for example, +DATE+, +TIME+, and +TIMESTAMP+) will be coerced to 28 | +STRING+ in Hive. The +NUMERIC+ and +DECIMAL+ SQL types will be coerced to 29 | +DOUBLE+. In these cases, Sqoop will emit a warning in its log messages 30 | informing you of the loss of precision. 31 | -------------------------------------------------------------------------------- /src/docs/dev/compiling.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | 21 | Compiling Sqoop from Source 22 | --------------------------- 23 | 24 | You can obtain the source code for Sqoop using following command: 25 | git clone https://git-wip-us.apache.org/repos/asf/sqoop.git 26 | 27 | Sqoop source code is held in a +git+ repository. Instructions for 28 | retrieving source from the repository are provided at: 29 | TODO provide a page in the web site. 30 | 31 | Compilation instructions are provided in the +COMPILING.txt+ file in 32 | the root of the source repository. 33 | 34 | -------------------------------------------------------------------------------- /src/docs/user/version.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | 21 | +sqoop-version+ 22 | --------------- 23 | 24 | Purpose 25 | ~~~~~~~ 26 | 27 | Display version information for Sqoop. 28 | 29 | Syntax 30 | ~~~~~~ 31 | 32 | ---- 33 | $ sqoop version 34 | $ sqoop-version 35 | ---- 36 | 37 | 38 | Example Invocations 39 | ~~~~~~~~~~~~~~~~~~~ 40 | 41 | Display the version: 42 | 43 | ---- 44 | $ sqoop version 45 | Sqoop {revnumber} 46 | git commit id 46b3e06b79a8411320d77c984c3030db47dd1c22 47 | Compiled by aaron@jargon on Mon May 17 13:43:22 PDT 2010 48 | ---- 49 | 50 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/manager/oracle/util/BinaryFloatGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.manager.oracle.util; 20 | 21 | /** 22 | * Generates Float test data. Test data is distributed over the entire range of 23 | * possible floats, including NaN, positive and negative infinity and positive 24 | * and negative zero. 25 | */ 26 | public class BinaryFloatGenerator extends OraOopTestDataGenerator { 27 | @Override 28 | public Float next() { 29 | return Float.intBitsToFloat(rng.nextInt()); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/util/ErrorableThread.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.util; 20 | 21 | /** 22 | * A thread which has an error bit which can be set from within the thread. 23 | */ 24 | public abstract class ErrorableThread extends Thread { 25 | 26 | private volatile boolean error; 27 | 28 | public ErrorableThread() { 29 | this.error = false; 30 | } 31 | 32 | protected void setError() { 33 | this.error = true; 34 | } 35 | 36 | public boolean isErrored() { 37 | return this.error; 38 | } 39 | 40 | } 41 | 42 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/manager/oracle/util/BinaryDoubleGenerator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.manager.oracle.util; 20 | 21 | /** 22 | * Generates Double test data. Test data is distributed over the entire range of 23 | * possible doubles, including NaN, positive and negative infinity and positive 24 | * and negative zero. 25 | */ 26 | public class BinaryDoubleGenerator extends OraOopTestDataGenerator { 27 | @Override 28 | public Double next() { 29 | return Double.longBitsToDouble(rng.nextLong()); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/validation/ValidationException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.validation; 20 | 21 | /** 22 | * An implementation of Exception that is used to propagate 23 | * validation related errors or failures. 24 | */ 25 | public class ValidationException extends Exception { 26 | 27 | public ValidationException(String message) { 28 | super(message); 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | String msg = getMessage(); 34 | return (null == msg) ? "ValidationException" : msg; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/lib/FieldMappable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.lib; 19 | 20 | import java.util.Map; 21 | 22 | /** 23 | * Interface describing a class capable of returning a map of the fields 24 | * of the object to their values. 25 | */ 26 | public interface FieldMappable { 27 | 28 | /** 29 | * Returns a map containing all fields of this record. 30 | * @return a map from column names to the object-based values for 31 | * this record. The map may not be null, though it may be empty. 32 | */ 33 | Map getFieldMap(); 34 | } 35 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/util/ErrorableAsyncSink.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.util; 20 | 21 | /** 22 | * Partial implementation of AsyncSink that relies on ErrorableThread to 23 | * provide a status bit for the join() method. 24 | */ 25 | public abstract class ErrorableAsyncSink extends AsyncSink { 26 | 27 | protected ErrorableThread child; 28 | 29 | public int join() throws InterruptedException { 30 | child.join(); 31 | if (child.isErrored()) { 32 | return 1; 33 | } else { 34 | return 0; 35 | } 36 | } 37 | 38 | } 39 | 40 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/db/decorator/JdbcConnectionFactoryDecorator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.db.decorator; 20 | 21 | import org.apache.sqoop.db.JdbcConnectionFactory; 22 | 23 | public abstract class JdbcConnectionFactoryDecorator implements JdbcConnectionFactory { 24 | 25 | protected final JdbcConnectionFactory decorated; 26 | 27 | public JdbcConnectionFactoryDecorator(JdbcConnectionFactory decorated) { 28 | this.decorated = decorated; 29 | } 30 | 31 | public JdbcConnectionFactory getDecorated() { 32 | return decorated; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/io/UnsupportedCodecException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.io; 19 | 20 | import java.io.IOException; 21 | 22 | /** 23 | * Thrown when a compression codec cannot be recognized. 24 | */ 25 | public class UnsupportedCodecException extends IOException { 26 | 27 | public UnsupportedCodecException() { 28 | super("UnsupportedCodecException"); 29 | } 30 | 31 | public UnsupportedCodecException(String msg) { 32 | super(msg); 33 | } 34 | 35 | public UnsupportedCodecException(Throwable cause) { 36 | super(cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/docs/sip/README.txt: -------------------------------------------------------------------------------- 1 | 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | 18 | This directory contains the archive of accepted Sqoop Improvement Proposals 19 | (SIPs). An SIP describes a proposed modification to Sqoop in technical detail. 20 | An accepted SIP is a design document that has been integrated into Sqoop. 21 | This directory serves as a technical reference for Sqoop developers to 22 | understand previous design decisions. 23 | 24 | The SIP home is on the Sqoop development wiki at: 25 | TODO 26 | 27 | The wiki SIP home contains all SIPs including those which are in "proposal", 28 | "implementation available," or "rejected" state. 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/docs/user/import-all-tables-purpose.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | The +import-all-tables+ tool imports a set of tables from an RDBMS to HDFS. 21 | Data from each table is stored in a separate directory in HDFS. 22 | 23 | For the +import-all-tables+ tool to be useful, the following conditions 24 | must be met: 25 | 26 | - Each table must have a single-column primary key or +--autoreset-to-one-mapper+ option must be used. 27 | - You must intend to import all columns of each table. 28 | - You must not intend to use non-default splitting column, nor impose 29 | any conditions via a +WHERE+ clause. 30 | 31 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/tool/ToolPlugin.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.tool; 20 | 21 | import java.util.List; 22 | 23 | /** 24 | * Abstract base class that defines the ToolPlugin API; additional SqoopTool 25 | * implementations may be registered with the system via ToolPlugin classes. 26 | */ 27 | public abstract class ToolPlugin { 28 | /** 29 | * Describes the tools made available by this plugin. 30 | * @return a list of ToolDesc objects containing the tool name, class, 31 | * and description. 32 | */ 33 | public abstract List getTools(); 34 | } 35 | -------------------------------------------------------------------------------- /src/docs/user/export-purpose.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | The +export+ tool exports a set of files from HDFS back to an RDBMS. 21 | The target table must already exist in the database. The input files 22 | are read and parsed into a set of records according to the 23 | user-specified delimiters. 24 | 25 | The default operation is to transform these into a set of +INSERT+ 26 | statements that inject the records into the database. In "update mode," 27 | Sqoop will generate +UPDATE+ statements that replace existing records 28 | in the database, and in "call mode" Sqoop will make a stored procedure 29 | call for each record. 30 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/MergeRecordMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce; 20 | 21 | import java.io.IOException; 22 | import org.apache.hadoop.io.LongWritable; 23 | import org.apache.sqoop.lib.SqoopRecord; 24 | 25 | /** 26 | * Mapper for the merge program which operates on SequenceFiles. 27 | */ 28 | public class MergeRecordMapper 29 | extends MergeMapperBase { 30 | 31 | public void map(LongWritable key, SqoopRecord val, Context c) 32 | throws IOException, InterruptedException { 33 | processRecord(val, c); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/docs/man/sqoop-help.txt: -------------------------------------------------------------------------------- 1 | sqoop-help(1) 2 | ============= 3 | 4 | NAME 5 | ---- 6 | sqoop-help - List available tools and display usage information 7 | 8 | SYNOPSIS 9 | -------- 10 | 'sqoop-help' [tool-name] 11 | 12 | 'sqoop help' [tool-name] 13 | 14 | 15 | DESCRIPTION 16 | ----------- 17 | 18 | List tools available in Sqoop and explain their usage. 19 | 20 | OPTIONS 21 | ------- 22 | 23 | With no arguments, displays the list of available tools. With an optional 24 | tool name specified, displays the usage for that particular tool. 25 | 26 | //// 27 | Copyright 2011 The Apache Software Foundation 28 | 29 | Licensed to the Apache Software Foundation (ASF) under one 30 | or more contributor license agreements. See the NOTICE file 31 | distributed with this work for additional information 32 | regarding copyright ownership. The ASF licenses this file 33 | to you under the Apache License, Version 2.0 (the 34 | "License"); you may not use this file except in compliance 35 | with the License. You may obtain a copy of the License at 36 | 37 | http://www.apache.org/licenses/LICENSE-2.0 38 | 39 | Unless required by applicable law or agreed to in writing, software 40 | distributed under the License is distributed on an "AS IS" BASIS, 41 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 42 | See the License for the specific language governing permissions and 43 | limitations under the License. 44 | //// 45 | 46 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/validation/ValidationFailureHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.validation; 20 | 21 | /** 22 | * This is the primary interface that dictates as to 23 | * how the validation failures are handled. 24 | */ 25 | public interface ValidationFailureHandler { 26 | 27 | /** 28 | * Method that handles the validation failure. 29 | * 30 | * @param validationContext validation context 31 | * @return if failure was handled or not 32 | * @throws ValidationException 33 | */ 34 | boolean handle(ValidationContext validationContext) 35 | throws ValidationException; 36 | } 37 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3TextImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestTextImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3TextImport extends AbstractTestTextImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3TextImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3AvroImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestAvroImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3AvroImport extends AbstractTestAvroImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3AvroImport() { 33 | super(credentialsRule); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/hive/minicluster/AuthenticationConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.hive.minicluster; 20 | 21 | import org.apache.sqoop.db.JdbcConnectionFactory; 22 | 23 | import java.security.PrivilegedAction; 24 | import java.util.Map; 25 | 26 | public interface AuthenticationConfiguration { 27 | 28 | Map getAuthenticationConfig(); 29 | 30 | String getUrlParams(); 31 | 32 | T doAsAuthenticated(PrivilegedAction action); 33 | 34 | void init(); 35 | 36 | JdbcConnectionFactory decorateConnectionFactory(JdbcConnectionFactory connectionFactory); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/splitby/MysqlSplitByImportTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob.splitby; 20 | 21 | import org.apache.sqoop.testcategories.thirdpartytest.MysqlTest; 22 | import org.apache.sqoop.testutil.adapter.DatabaseAdapter; 23 | import org.apache.sqoop.testutil.adapter.MysqlDatabaseAdapter; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(MysqlTest.class) 27 | public class MysqlSplitByImportTest extends SplitByImportTestBase { 28 | 29 | @Override 30 | public DatabaseAdapter createAdapter() { 31 | return new MysqlDatabaseAdapter(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/splitby/OracleSplitByImportTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob.splitby; 20 | 21 | import org.apache.sqoop.testcategories.thirdpartytest.OracleTest; 22 | import org.apache.sqoop.testutil.adapter.DatabaseAdapter; 23 | import org.apache.sqoop.testutil.adapter.OracleDatabaseAdapter; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(OracleTest.class) 27 | public class OracleSplitByImportTest extends SplitByImportTestBase { 28 | 29 | @Override 30 | public DatabaseAdapter createAdapter() { 31 | return new OracleDatabaseAdapter(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/docs/user/create-hive-table-purpose.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | The +create-hive-table+ tool populates a Hive metastore with a 21 | definition for a table based on a database table previously imported 22 | to HDFS, or one planned to be imported. This effectively performs the 23 | "+\--hive-import+" step of +sqoop-import+ without running the 24 | preceeding import. 25 | 26 | If data was already loaded to HDFS, you can use this tool to finish 27 | the pipeline of importing the data to Hive. You can also create Hive tables 28 | with this tool; data then can be imported and populated into 29 | the target after a preprocessing step run by the user. 30 | 31 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/parquet/ParquetConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | package org.apache.sqoop.mapreduce.parquet; 21 | 22 | public final class ParquetConstants { 23 | 24 | public static final String SQOOP_PARQUET_AVRO_SCHEMA_KEY = "parquetjob.avro.schema"; 25 | 26 | public static final String SQOOP_PARQUET_OUTPUT_CODEC_KEY = "parquetjob.output.codec"; 27 | 28 | public static final String PARQUET_JOB_CONFIGURATOR_IMPLEMENTATION_KEY = "parquetjob.configurator.implementation"; 29 | 30 | private ParquetConstants() { 31 | throw new AssertionError("This class is meant for static use only."); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3ParquetImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestParquetImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3ParquetImport extends AbstractTestParquetImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3ParquetImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/manager/oracle/util/HadoopFiles.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.manager.oracle.util; 20 | 21 | import org.apache.hadoop.conf.Configuration; 22 | import org.apache.hadoop.fs.FileSystem; 23 | import org.apache.hadoop.fs.Path; 24 | 25 | /** 26 | * Helper class for HDFS related functions. 27 | */ 28 | public final class HadoopFiles { 29 | private HadoopFiles() { 30 | } 31 | 32 | public static void delete(Path file, boolean recursive) throws Exception { 33 | FileSystem fileSystem = FileSystem.get(file.toUri(), new Configuration()); 34 | fileSystem.delete(file, recursive); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /conf/sqoop-env-template.sh: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # included in all the hadoop scripts with source command 17 | # should not be executable directly 18 | # also should not be passed any arguments, since we need original $* 19 | 20 | # Set Hadoop-specific environment variables here. 21 | 22 | #Set path to where bin/hadoop is available 23 | #export HADOOP_COMMON_HOME= 24 | 25 | #Set path to where hadoop-*-core.jar is available 26 | #export HADOOP_MAPRED_HOME= 27 | 28 | #set the path to where bin/hbase is available 29 | #export HBASE_HOME= 30 | 31 | #Set the path to where bin/hive is available 32 | #export HIVE_HOME= 33 | 34 | #Set the path for where zookeper config dir is 35 | #export ZOOCFGDIR= 36 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/splitby/PostgresSplitByImportTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob.splitby; 20 | 21 | import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest; 22 | import org.apache.sqoop.testutil.adapter.DatabaseAdapter; 23 | import org.apache.sqoop.testutil.adapter.PostgresDatabaseAdapter; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(PostgresqlTest.class) 27 | public class PostgresSplitByImportTest extends SplitByImportTestBase { 28 | 29 | @Override 30 | public DatabaseAdapter createAdapter() { 31 | return new PostgresDatabaseAdapter(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/importjob/splitby/SqlServerSplitByImportTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.importjob.splitby; 20 | 21 | import org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest; 22 | import org.apache.sqoop.testutil.adapter.DatabaseAdapter; 23 | import org.apache.sqoop.testutil.adapter.SqlServerDatabaseAdapter; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(SqlServerTest.class) 27 | public class SqlServerSplitByImportTest extends SplitByImportTestBase { 28 | 29 | @Override 30 | public DatabaseAdapter createAdapter() { 31 | return new SqlServerDatabaseAdapter(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3SequenceFileImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestSequenceFileImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3SequenceFileImport extends AbstractTestSequenceFileImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3SequenceFileImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/util/StoredAsProperty.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.util; 20 | 21 | import java.lang.annotation.Documented; 22 | import java.lang.annotation.ElementType; 23 | import java.lang.annotation.Retention; 24 | import java.lang.annotation.RetentionPolicy; 25 | import java.lang.annotation.Target; 26 | 27 | /** 28 | * Used by SqoopOptions to denote that a field is stored in a particular 29 | * named property when reifying the object's state to permanent storage. 30 | */ 31 | @Documented 32 | @Retention(RetentionPolicy.RUNTIME) 33 | @Target(ElementType.FIELD) 34 | public @interface StoredAsProperty { 35 | String value(); 36 | } 37 | 38 | -------------------------------------------------------------------------------- /src/docs/user/output-formatting-args.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | --fields-terminated-by (char):: 21 | Sets the field separator character 22 | 23 | --lines-terminated-by (char):: 24 | Sets the end-of-line character 25 | 26 | --optionally-enclosed-by (char):: 27 | Sets a field-enclosing character which may be used if a 28 | value contains delimiter characters. 29 | 30 | --enclosed-by (char):: 31 | Sets a field-enclosing character which will be used for all fields. 32 | 33 | --escaped-by (char):: 34 | Sets the escape character 35 | 36 | --mysql-delimiters:: 37 | Uses MySQL's default delimiter set: 38 | + 39 | fields: , lines: \n escaped-by: \ optionally-enclosed-by: ' 40 | 41 | -------------------------------------------------------------------------------- /src/docs/man/sqoop-list-tables.txt: -------------------------------------------------------------------------------- 1 | sqoop-list-tables(1) 2 | ==================== 3 | 4 | NAME 5 | ---- 6 | sqoop-list-tables - List tables available for import in a database. 7 | 8 | SYNOPSIS 9 | -------- 10 | 'sqoop-list-tables' 11 | 12 | 'sqoop list-tables' 13 | 14 | 15 | DESCRIPTION 16 | ----------- 17 | 18 | List tables present in a database. 19 | 20 | OPTIONS 21 | ------- 22 | 23 | The +--connect+ option is required. 24 | 25 | include::common-args.txt[] 26 | 27 | 28 | ENVIRONMENT 29 | ----------- 30 | 31 | See 'sqoop(1)' 32 | 33 | //// 34 | Copyright 2011 The Apache Software Foundation 35 | 36 | Licensed to the Apache Software Foundation (ASF) under one 37 | or more contributor license agreements. See the NOTICE file 38 | distributed with this work for additional information 39 | regarding copyright ownership. The ASF licenses this file 40 | to you under the Apache License, Version 2.0 (the 41 | "License"); you may not use this file except in compliance 42 | with the License. You may obtain a copy of the License at 43 | 44 | http://www.apache.org/licenses/LICENSE-2.0 45 | 46 | Unless required by applicable law or agreed to in writing, software 47 | distributed under the License is distributed on an "AS IS" BASIS, 48 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 49 | See the License for the specific language governing permissions and 50 | limitations under the License. 51 | //// 52 | 53 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/hbase/HBaseImportTypesTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.hbase; 20 | 21 | import java.io.IOException; 22 | 23 | import org.junit.Test; 24 | 25 | /** 26 | * 27 | */ 28 | public class HBaseImportTypesTest extends HBaseTestCase { 29 | 30 | @Test 31 | public void testStrings() throws IOException { 32 | String [] argv = getArgv(true, "stringT", "stringF", true, null); 33 | String [] types = { "INT", "VARCHAR(32)" }; 34 | String [] vals = { "0", "'abc'" }; 35 | createTableWithColTypes(types, vals); 36 | runImport(argv); 37 | verifyHBaseCell("stringT", "0", "stringF", getColName(1), "abc"); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/manager/oracle/OracleActiveInstance.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.manager.oracle; 20 | 21 | /** 22 | * Wraps data from v$active_instances. 23 | */ 24 | public class OracleActiveInstance { 25 | 26 | private String instanceName; 27 | private String hostName; 28 | 29 | public String getInstanceName() { 30 | return instanceName; 31 | } 32 | 33 | public void setInstanceName(String newInstanceName) { 34 | this.instanceName = newInstanceName; 35 | } 36 | 37 | public String getHostName() { 38 | return hostName; 39 | } 40 | 41 | public void setHostName(String newHostName) { 42 | this.hostName = newHostName; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableTextExportMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce.db.netezza; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.hadoop.io.LongWritable; 24 | import org.apache.hadoop.io.Text; 25 | 26 | /** 27 | * Netezza export mapper for text records. 28 | */ 29 | public class NetezzaExternalTableTextExportMapper extends 30 | NetezzaExternalTableExportMapper { 31 | 32 | @Override 33 | public void map(LongWritable key, Text text, Context context) 34 | throws IOException, InterruptedException { 35 | writeTextRecord(text); 36 | context.progress(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/docs/user/support.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | Getting Support 21 | --------------- 22 | 23 | Some general information is available at the 24 | http://sqoop.apache.org/ 25 | 26 | Report bugs in Sqoop to the issue tracker at 27 | https://issues.apache.org/jira/browse/SQOOP. 28 | 29 | Questions and discussion regarding the usage of Sqoop should be directed to the 30 | http://sqoop.apache.org/mail-lists.html[sqoop-user mailing list]. 31 | 32 | Before contacting either forum, run your Sqoop job with the 33 | +\--verbose+ flag to acquire as much debugging information as 34 | possible. Also report the string returned by +sqoop version+ as 35 | well as the version of Hadoop you are running (+hadoop version+). 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3IncrementalMergeTextImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestIncrementalMergeTextImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3IncrementalMergeTextImport extends AbstractTestIncrementalMergeTextImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3IncrementalMergeTextImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/testutil/InjectableManagerFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.testutil; 20 | 21 | import org.apache.sqoop.manager.ConnManager; 22 | import org.apache.sqoop.manager.ManagerFactory; 23 | import org.apache.sqoop.metastore.JobData; 24 | import org.apache.sqoop.testutil.InjectableConnManager; 25 | 26 | /** 27 | * ManagerFactory that is used for testing; this accepts any 28 | * connection string. 29 | */ 30 | public class InjectableManagerFactory extends ManagerFactory { 31 | 32 | public ConnManager accept(JobData data) { 33 | // Always accept and use the injectable manager. 34 | return new InjectableConnManager(data.getSqoopOptions()); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/parquet/hadoop/HadoopMergeParquetReducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce.parquet.hadoop; 20 | 21 | import org.apache.avro.generic.GenericRecord; 22 | import org.apache.sqoop.mapreduce.MergeParquetReducer; 23 | 24 | import java.io.IOException; 25 | 26 | /** 27 | * An implementation of {@link MergeParquetReducer} which depends on the Hadoop Parquet library. 28 | */ 29 | public class HadoopMergeParquetReducer extends MergeParquetReducer { 30 | 31 | @Override 32 | protected void write(Context context, GenericRecord record) throws IOException, InterruptedException { 33 | context.write(null, record); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3IncrementalAppendAvroImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestIncrementalAppendAvroImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3IncrementalAppendAvroImport extends AbstractTestIncrementalAppendAvroImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3IncrementalAppendAvroImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3IncrementalAppendTextImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestIncrementalAppendTextImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3IncrementalAppendTextImport extends AbstractTestIncrementalAppendTextImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3IncrementalAppendTextImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /conf/sqoop-env-template.cmd: -------------------------------------------------------------------------------- 1 | @echo off 2 | :: Licensed to the Apache Software Foundation (ASF) under one or more 3 | :: contributor license agreements. See the NOTICE file distributed with 4 | :: this work for additional information regarding copyright ownership. 5 | :: The ASF licenses this file to You under the Apache License, Version 2.0 6 | :: (the "License"); you may not use this file except in compliance with 7 | :: the License. You may obtain a copy of the License at 8 | :: 9 | :: http://www.apache.org/licenses/LICENSE-2.0 10 | :: 11 | :: Unless required by applicable law or agreed to in writing, software 12 | :: distributed under the License is distributed on an "AS IS" BASIS, 13 | :: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | :: See the License for the specific language governing permissions and 15 | :: limitations under the License. 16 | 17 | :: included in all the hadoop scripts with source command 18 | :: should not be executable directly 19 | :: also should not be passed any arguments, since we need original $* 20 | 21 | :: Set Hadoop-specific environment variables here. 22 | 23 | ::Set path to where bin/hadoop is available 24 | ::set HADOOP_COMMON_HOME= 25 | 26 | ::Set path to where hadoop-*-core.jar is available 27 | ::set HADOOP_MAPRED_HOME= 28 | 29 | ::set the path to where bin/hbase is available 30 | ::set HBASE_HOME= 31 | 32 | ::Set the path to where bin/hive is available 33 | ::set HIVE_HOME= 34 | 35 | ::Set the path for where zookeper config dir is 36 | ::set ZOOCFGDIR= 37 | -------------------------------------------------------------------------------- /src/docs/web/footer.xsl: -------------------------------------------------------------------------------- 1 | 5 | 6 | 23 | 24 | 25 | 26 | 27 |

28 | Documentation Home 30 |
31 | This document was built from Sqoop source available at 32 | https://git-wip-us.apache.org/repos/asf?p=sqoop.git. 33 |
34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/manager/oracle/OraOopOracleDataChunk.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.manager.oracle; 20 | 21 | import org.apache.hadoop.io.Writable; 22 | 23 | /** 24 | * How data should be split between mappers. 25 | */ 26 | public abstract class OraOopOracleDataChunk implements Writable { 27 | 28 | private String id; 29 | 30 | public abstract long getNumberOfBlocks(); 31 | 32 | public String getWhereClause() { 33 | return "1=1"; 34 | } 35 | 36 | public String getPartitionClause() { 37 | return ""; 38 | } 39 | 40 | public String getId() { 41 | return id; 42 | } 43 | 44 | public void setId(String newId) { 45 | this.id = newId; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/docs/user/controlling-output-format.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | 21 | Controlling the Output Format 22 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 23 | include::output-formatting.txt[] 24 | 25 | The following arguments allow you to control the output format of 26 | records: 27 | 28 | include::output-formatting-args.txt[] 29 | 30 | For example, we may want to separate records by tab characters, with 31 | every record surrounded by "double quotes", and internal quote marks 32 | escaped by a backslash (+\+) character: 33 | 34 | ---- 35 | $ sqoop --connect jdbc:mysql://database.example.com/employees \ 36 | --table employee_names --fields-terminated-by \t \ 37 | --lines-terminated-by \n --enclosed-by '\"' --escaped-by '\\' 38 | ---- 39 | 40 | -------------------------------------------------------------------------------- /src/docs/user/distributed-cache.txt: -------------------------------------------------------------------------------- 1 | //// 2 | Licensed to the Apache Software Foundation (ASF) under one 3 | or more contributor license agreements. See the NOTICE file 4 | distributed with this work for additional information 5 | regarding copyright ownership. The ASF licenses this file 6 | to you under the Apache License, Version 2.0 (the 7 | "License"); you may not use this file except in compliance 8 | with the License. You may obtain a copy of the License at 9 | 10 | http://www.apache.org/licenses/LICENSE-2.0 11 | 12 | Unless required by applicable law or agreed to in writing, software 13 | distributed under the License is distributed on an "AS IS" BASIS, 14 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | See the License for the specific language governing permissions and 16 | limitations under the License. 17 | //// 18 | 19 | Controlling Distributed Cache 20 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 21 | 22 | Sqoop will copy the jars in $SQOOP_HOME/lib folder to job cache every 23 | time when start a Sqoop job. When launched by Oozie this is unnecessary 24 | since Oozie use its own Sqoop share lib which keeps Sqoop dependencies 25 | in the distributed cache. Oozie will do the localization on each 26 | worker node for the Sqoop dependencies only once during the first Sqoop 27 | job and reuse the jars on worker node for subsquencial jobs. Using 28 | option +--skip-dist-cache+ in Sqoop command when launched by Oozie will 29 | skip the step which Sqoop copies its dependencies to job cache and save 30 | massive I/O. 31 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/db/netezza/NetezzaExternalTableRecordExportMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce.db.netezza; 20 | 21 | import java.io.IOException; 22 | 23 | import org.apache.hadoop.io.LongWritable; 24 | import org.apache.sqoop.lib.SqoopRecord; 25 | 26 | /** 27 | * Netezza export mapper for Sqoop records. 28 | */ 29 | public class NetezzaExternalTableRecordExportMapper extends 30 | NetezzaExternalTableExportMapper { 31 | 32 | @Override 33 | public void map(LongWritable key, SqoopRecord sqr, Context context) 34 | throws IOException, InterruptedException { 35 | writeSqoopRecord(sqr); 36 | context.progress(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/mainframe/MainframeDatasetImportMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce.mainframe; 20 | 21 | import org.apache.hadoop.io.Text; 22 | import org.apache.sqoop.lib.SqoopRecord; 23 | 24 | /** 25 | * Mapper that writes mainframe dataset records in Text format to multiple files 26 | * based on the key, which is the index of the datasets in the input split. 27 | */ 28 | public class MainframeDatasetImportMapper extends AbstractMainframeDatasetImportMapper { 29 | 30 | @Override 31 | protected Text createOutKey(SqoopRecord sqoopRecord) { 32 | Text result = new Text(); 33 | result.set(sqoopRecord.toString()); 34 | return result; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3IncrementalMergeParquetImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestIncrementalMergeParquetImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3IncrementalMergeParquetImport extends AbstractTestIncrementalMergeParquetImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3IncrementalMergeParquetImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/scripts/run-perftest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2011 The Apache Software Foundation 4 | # 5 | # Licensed to the Apache Software Foundation (ASF) under one 6 | # or more contributor license agreements. See the NOTICE file 7 | # distributed with this work for additional information 8 | # regarding copyright ownership. The ASF licenses this file 9 | # to you under the Apache License, Version 2.0 (the 10 | # "License"); you may not use this file except in compliance 11 | # with the License. You may obtain a copy of the License at 12 | # 13 | # http://www.apache.org/licenses/LICENSE-2.0 14 | # 15 | # Unless required by applicable law or agreed to in writing, software 16 | # distributed under the License is distributed on an "AS IS" BASIS, 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | # See the License for the specific language governing permissions and 19 | # limitations under the License. 20 | 21 | # run-perftest.sh 22 | # USAGE: 23 | # ./run-perftest.sh PerfTestClassName [arg arg arg...] 24 | # 25 | # This script will run one of the classes from src/perftest/ with the 26 | # correct classpath set up. 27 | 28 | bin=`dirname $0` 29 | bin=`cd ${bin} && pwd` 30 | 31 | # This is run in src/scripts/ 32 | SQOOP_HOME="${bin}/../../" 33 | 34 | # Set up environment and classpath 35 | source ${SQOOP_HOME}/bin/configure-sqoop "${bin}" 36 | 37 | PERFTEST_CLASSES=${SQOOP_HOME}/build/perftest/classes 38 | 39 | export HADOOP_CLASSPATH=${PERFTEST_CLASSES}:${SQOOP_JAR}:${HADOOP_CLASSPATH} 40 | ${HADOOP_COMMON_HOME}/bin/hadoop "$@" 41 | 42 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3IncrementalAppendParquetImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestIncrementalAppendParquetImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3IncrementalAppendParquetImport extends AbstractTestIncrementalAppendParquetImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3IncrementalAppendParquetImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/manager/oracle/OracleTableColumns.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.manager.oracle; 20 | 21 | import java.util.Iterator; 22 | 23 | /** 24 | * Contains a list of Oracle columns. 25 | */ 26 | public class OracleTableColumns extends 27 | OraOopGenerics.ObjectList { 28 | 29 | public OracleTableColumn findColumnByName(String columnName) { 30 | 31 | OracleTableColumn result; 32 | 33 | Iterator iterator = this.iterator(); 34 | while (iterator.hasNext()) { 35 | result = iterator.next(); 36 | if (result.getName().equals(columnName)) { 37 | return result; 38 | } 39 | } 40 | return null; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/tool/VersionTool.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.tool; 20 | 21 | import org.apache.sqoop.SqoopOptions; 22 | import org.apache.sqoop.cli.ToolOptions; 23 | 24 | /** 25 | * Tool that prints Sqoop's version. 26 | */ 27 | public class VersionTool extends BaseSqoopTool { 28 | 29 | public VersionTool() { 30 | super("version"); 31 | } 32 | 33 | @Override 34 | /** {@inheritDoc} */ 35 | public int run(SqoopOptions options) { 36 | System.out.print(new org.apache.sqoop.SqoopVersion().toString()); 37 | return 0; 38 | } 39 | 40 | @Override 41 | public void printHelp(ToolOptions opts) { 42 | System.out.println("usage: sqoop " + getToolName()); 43 | } 44 | } 45 | 46 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/parquet/ParquetMergeJobConfigurator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce.parquet; 20 | 21 | import org.apache.hadoop.conf.Configuration; 22 | import org.apache.hadoop.fs.Path; 23 | import org.apache.hadoop.mapreduce.Job; 24 | 25 | import java.io.IOException; 26 | 27 | /** 28 | * This interface defines the type of a product of {@link ParquetJobConfiguratorFactory}. 29 | * The implementations of the methods of this interface help to configure Sqoop Parquet merge jobs. 30 | */ 31 | public interface ParquetMergeJobConfigurator { 32 | 33 | void configureParquetMergeJob(Configuration conf, Job job, Path oldPath, Path newPath, Path finalPath) throws IOException; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/db/OracleDateSplitter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.mapreduce.db; 19 | 20 | import java.util.Date; 21 | 22 | /** 23 | * Implement DBSplitter over date/time values returned by an Oracle db. 24 | * Make use of logic from DateSplitter, since this just needs to use 25 | * some Oracle-specific functions on the formatting end when generating 26 | * InputSplits. 27 | */ 28 | public class OracleDateSplitter extends DateSplitter { 29 | 30 | @SuppressWarnings("unchecked") 31 | @Override 32 | protected String dateToString(Date d) { 33 | // Oracle Data objects are always actually Timestamps 34 | return "TO_TIMESTAMP('" + d.toString() + "', 'YYYY-MM-DD HH24:MI:SS.FF')"; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/metastore/hsqldb/HsqldbJobToolTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.metastore.hsqldb; 20 | 21 | import org.apache.sqoop.metastore.JobToolTestBase; 22 | 23 | /** 24 | * Test that the Job Tool works in Hsqldb 25 | * 26 | * This class is named in such a way that Sqoop's default QA process does 27 | * not run it. You need to run this manually with 28 | * -Dtestcase=HsqldbJobToolTest or -Dthirdparty=true. 29 | * 30 | * This uses JDBC to store and retrieve metastore data from a local Hsqldb server 31 | */ 32 | 33 | public class HsqldbJobToolTest extends JobToolTestBase { 34 | 35 | public HsqldbJobToolTest() { 36 | super( "jdbc:hsqldb:mem:sqoopmetastore", "SA" , ""); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/docs/man/sqoop-metastore.txt: -------------------------------------------------------------------------------- 1 | sqoop-metastore(1) 2 | ================== 3 | 4 | NAME 5 | ---- 6 | sqoop-metastore - Host a shared repository for saved Sqoop jobs. 7 | 8 | SYNOPSIS 9 | -------- 10 | 'sqoop-metastore' 11 | 12 | 'sqoop metastore' 13 | 14 | 15 | DESCRIPTION 16 | ----------- 17 | 18 | include::../user/metastore-purpose.txt[] 19 | 20 | OPTIONS 21 | ------- 22 | 23 | Metastore management options 24 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 25 | 26 | --shutdown:: 27 | Shuts down a running metastore instance on the same machine 28 | 29 | ENVIRONMENT 30 | ----------- 31 | 32 | See 'sqoop(1)' 33 | 34 | 35 | //// 36 | Copyright 2011 The Apache Software Foundation 37 | 38 | Licensed to the Apache Software Foundation (ASF) under one 39 | or more contributor license agreements. See the NOTICE file 40 | distributed with this work for additional information 41 | regarding copyright ownership. The ASF licenses this file 42 | to you under the Apache License, Version 2.0 (the 43 | "License"); you may not use this file except in compliance 44 | with the License. You may obtain a copy of the License at 45 | 46 | http://www.apache.org/licenses/LICENSE-2.0 47 | 48 | Unless required by applicable law or agreed to in writing, software 49 | distributed under the License is distributed on an "AS IS" BASIS, 50 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 51 | See the License for the specific language governing permissions and 52 | limitations under the License. 53 | //// 54 | 55 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/cloud/s3/TestS3IncrementalAppendSequenceFileImport.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cloud.s3; 20 | 21 | import org.apache.sqoop.cloud.AbstractTestIncrementalAppendSequenceFileImport; 22 | import org.apache.sqoop.testcategories.thirdpartytest.S3Test; 23 | import org.junit.ClassRule; 24 | import org.junit.experimental.categories.Category; 25 | 26 | @Category(S3Test.class) 27 | public class TestS3IncrementalAppendSequenceFileImport extends AbstractTestIncrementalAppendSequenceFileImport { 28 | 29 | @ClassRule 30 | public static S3CredentialsRule credentialsRule = new S3CredentialsRule(); 31 | 32 | public TestS3IncrementalAppendSequenceFileImport() { 33 | super(credentialsRule); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/cli/RelatedOptions.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.cli; 20 | 21 | import org.apache.commons.cli.Options; 22 | 23 | /** 24 | * Class that extends a set of options to provide a title for a 25 | * related set of options. 26 | */ 27 | public class RelatedOptions extends Options { 28 | 29 | private String relatedTitle; 30 | 31 | public RelatedOptions() { 32 | this(""); 33 | } 34 | 35 | public RelatedOptions(final String title) { 36 | super(); 37 | this.relatedTitle = title; 38 | } 39 | 40 | public String getTitle() { 41 | return relatedTitle; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return relatedTitle + "\n" + super.toString(); 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/mainframe/MainframeDatasetType.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce.mainframe; 20 | 21 | /******** 22 | * Basic enumeration for Mainframe data set types 23 | ********/ 24 | 25 | public enum MainframeDatasetType { 26 | GDG, PARTITIONED, SEQUENTIAL; 27 | 28 | @Override 29 | public String toString() { 30 | switch(this) { 31 | case GDG: return MainframeConfiguration.MAINFRAME_INPUT_DATASET_TYPE_GDG; 32 | case PARTITIONED: return MainframeConfiguration.MAINFRAME_INPUT_DATASET_TYPE_PARTITIONED; 33 | case SEQUENTIAL: return MainframeConfiguration.MAINFRAME_INPUT_DATASET_TYPE_SEQUENTIAL; 34 | default: throw new IllegalArgumentException(); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/perftest/README-perftest.txt: -------------------------------------------------------------------------------- 1 | 2 | Copyright 2011 The Apache Software Foundation 3 | 4 | Licensed to the Apache Software Foundation (ASF) under one 5 | or more contributor license agreements. See the NOTICE file 6 | distributed with this work for additional information 7 | regarding copyright ownership. The ASF licenses this file 8 | to you under the Apache License, Version 2.0 (the 9 | "License"); you may not use this file except in compliance 10 | with the License. You may obtain a copy of the License at 11 | 12 | http://www.apache.org/licenses/LICENSE-2.0 13 | 14 | Unless required by applicable law or agreed to in writing, software 15 | distributed under the License is distributed on an "AS IS" BASIS, 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | See the License for the specific language governing permissions and 18 | limitations under the License. 19 | 20 | 21 | = Performance and Stress Tests 22 | 23 | The files in this directory represent performance or stress tests of aspects 24 | of Sqoop. These are intended to be run by a developer as a part of a lengthy 25 | QA process but are not the "primary" tests which are expected to be run for 26 | every build. 27 | 28 | 29 | == Compiling 30 | 31 | To compile the tests in this directory, run 'ant jar compile-perf-test' in the 32 | project root. 33 | 34 | == Running 35 | 36 | After compiling the performance tests, you can run them with: 37 | 38 | $ src/scripts/run-perftest.sh [] 39 | 40 | e.g.: 41 | 42 | $ src/scripts/run-perftest.sh LobFileStressTest 43 | 44 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/manager/oracle/TestOracleTable.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.manager.oracle; 20 | 21 | import org.junit.Test; 22 | 23 | import static org.junit.Assert.assertEquals; 24 | 25 | /** 26 | * Unit tests for OracleTable. 27 | */ 28 | public class TestOracleTable extends OraOopTestCase { 29 | 30 | @Test 31 | public void testToString() { 32 | OracleTable table = new OracleTable("ORAOOP", "TEST_TABLE"); 33 | assertEquals("\"ORAOOP\".\"TEST_TABLE\"", table.toString()); 34 | 35 | table = new OracleTable("", "TEST_TABLE2"); 36 | assertEquals("\"TEST_TABLE2\"", table.toString()); 37 | 38 | table = new OracleTable("TEST_TABLE3"); 39 | assertEquals("\"TEST_TABLE3\"", table.toString()); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/docs/web/header.xsl: -------------------------------------------------------------------------------- 1 | 5 | 6 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 |

33 |
34 | Documentation Home 36 |
37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/MergeReducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce; 20 | 21 | import java.io.IOException; 22 | import org.apache.hadoop.io.NullWritable; 23 | import org.apache.hadoop.io.Text; 24 | import org.apache.hadoop.mapreduce.Reducer; 25 | import org.apache.sqoop.lib.SqoopRecord; 26 | 27 | /** 28 | * Reducer for merge tool. Given records tagged as 'old' or 'new', emit 29 | * a new one if possible; otherwise, an old one. 30 | */ 31 | public class MergeReducer 32 | extends MergeReducerBase { 33 | 34 | @Override 35 | protected void writeRecord(SqoopRecord record, Context c) 36 | throws IOException, InterruptedException { 37 | c.write(record, NullWritable.get()); 38 | } 39 | } 40 | 41 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/parquet/ParquetJobConfiguratorFactory.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | *

10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | *

12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce.parquet; 20 | 21 | /** 22 | * This interface is an abstract factory of objects which configure Sqoop Parquet jobs. 23 | * Every product is responsible for configuring different types of Sqoop jobs. 24 | * 25 | * @see ParquetImportJobConfigurator 26 | * @see ParquetExportJobConfigurator 27 | * @see ParquetMergeJobConfigurator 28 | */ 29 | public interface ParquetJobConfiguratorFactory { 30 | 31 | ParquetImportJobConfigurator createParquetImportJobConfigurator(); 32 | 33 | ParquetExportJobConfigurator createParquetExportJobConfigurator(); 34 | 35 | ParquetMergeJobConfigurator createParquetMergeJobConfigurator(); 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/test/oraoop/pkg_tst_product_gen.psk: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE 2 | PACKAGE "PKG_ODG_$TABLE_NAME" 3 | AS 4 | 5 | /** 6 | * Licensed to the Apache Software Foundation (ASF) under one 7 | * or more contributor license agreements. See the NOTICE file 8 | * distributed with this work for additional information 9 | * regarding copyright ownership. The ASF licenses this file 10 | * to you under the Apache License, Version 2.0 (the 11 | * "License"); you may not use this file except in compliance 12 | * with the License. You may obtain a copy of the License at 13 | * 14 | * http://www.apache.org/licenses/LICENSE-2.0 15 | * 16 | * Unless required by applicable law or agreed to in writing, software 17 | * distributed under the License is distributed on an "AS IS" BASIS, 18 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 19 | * See the License for the specific language governing permissions and 20 | * limitations under the License. 21 | */ 22 | 23 | TYPE typ_oraoop_datagen IS RECORD 24 | ( 25 | $COLUMN_LIST 26 | ); 27 | 28 | TYPE typ_oraoop_datagen_tab IS TABLE OF typ_oraoop_datagen; 29 | 30 | TYPE typ_parameters IS RECORD 31 | ( 32 | start_range NUMBER, 33 | end_range NUMBER 34 | ); 35 | 36 | TYPE typ_rc_parameters IS REF CURSOR 37 | RETURN typ_parameters; 38 | 39 | FUNCTION fnc_gen_data (i_parameters typ_rc_parameters) 40 | RETURN typ_oraoop_datagen_tab 41 | PIPELINED 42 | PARALLEL_ENABLE(PARTITION i_parameters BY ANY); 43 | 44 | PROCEDURE prc_load_table (i_degree NUMBER, i_rows_per_slave NUMBER); 45 | END; -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/HBaseImportMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce; 20 | 21 | import java.io.IOException; 22 | import org.apache.hadoop.io.LongWritable; 23 | import org.apache.hadoop.io.NullWritable; 24 | import org.apache.sqoop.lib.SqoopRecord; 25 | 26 | /** 27 | * Imports records by writing them to HBase via the DelegatingOutputFormat 28 | * and the HBasePutProcessor. 29 | */ 30 | public class HBaseImportMapper 31 | extends AutoProgressMapper 32 | { 33 | 34 | @Override 35 | public void map(LongWritable key, SqoopRecord val, Context context) 36 | throws IOException, InterruptedException { 37 | context.write(val, NullWritable.get()); 38 | } 39 | } 40 | 41 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/lib/FieldMapProcessor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | package org.apache.sqoop.lib; 19 | 20 | import java.io.IOException; 21 | 22 | /** 23 | * Interface implemented by classes that process FieldMappable objects. 24 | */ 25 | public interface FieldMapProcessor { 26 | 27 | /** 28 | * Allow arbitrary processing of a FieldMappable object. 29 | * @param record an object which can emit a map of its field names to values. 30 | * @throws IOException if the processor encounters an IO error when 31 | * operating on this object. 32 | * @throws ProcessingException if the FieldMapProcessor encounters 33 | * a general processing error when operating on this object. 34 | */ 35 | void accept(FieldMappable record) throws IOException, ProcessingException; 36 | } 37 | -------------------------------------------------------------------------------- /src/test/org/apache/sqoop/manager/db2/DB2TestUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.manager.db2; 20 | 21 | public class DB2TestUtils { 22 | 23 | public static final String HOST_URL = System.getProperty("sqoop.test.db2.connectstring.host_url", "jdbc:db2://localhost:50000"); 24 | public static final String DATABASE_NAME = System.getProperty("sqoop.test.db2.connectstring.database", "SQOOP"); 25 | public static final String DATABASE_USER = System.getProperty("sqoop.test.db2.connectstring.username", "DB2INST1"); 26 | public static final String DATABASE_PASSWORD = System.getProperty("sqoop.test.db2.connectstring.password", "Sqoop12345"); 27 | public static final String CONNECT_STRING = HOST_URL + "/" + DATABASE_NAME + ":currentSchema=" + DATABASE_USER +";"; 28 | } 29 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/AccumuloImportMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce; 20 | 21 | import java.io.IOException; 22 | import org.apache.hadoop.io.LongWritable; 23 | import org.apache.hadoop.io.NullWritable; 24 | import org.apache.sqoop.lib.SqoopRecord; 25 | 26 | /** 27 | * Imports records by writing them to Accumulo via the DelegatingOutputFormat 28 | * and the AccumuloMutationProcessor. 29 | */ 30 | public class AccumuloImportMapper 31 | extends AutoProgressMapper 32 | { 33 | 34 | @Override 35 | public void map(LongWritable key, SqoopRecord val, Context context) 36 | throws IOException, InterruptedException { 37 | context.write(val, NullWritable.get()); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/mapreduce/parquet/hadoop/HadoopParquetExportMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.mapreduce.parquet.hadoop; 20 | 21 | import org.apache.avro.generic.GenericRecord; 22 | import org.apache.hadoop.io.NullWritable; 23 | import org.apache.sqoop.mapreduce.GenericRecordExportMapper; 24 | 25 | import java.io.IOException; 26 | 27 | /** 28 | * An implementation of {@link GenericRecordExportMapper} which depends on the Hadoop Parquet library. 29 | */ 30 | public class HadoopParquetExportMapper extends GenericRecordExportMapper { 31 | 32 | @Override 33 | protected void map(Void key, GenericRecord val, Context context) throws IOException, InterruptedException { 34 | context.write(toSqoopRecord(val), NullWritable.get()); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/docs/man/sqoop-eval.txt: -------------------------------------------------------------------------------- 1 | sqoop-eval(1) 2 | ============= 3 | 4 | NAME 5 | ---- 6 | sqoop-eval - Evalute a SQL query and display its results 7 | 8 | SYNOPSIS 9 | -------- 10 | 'sqoop-eval' 11 | 12 | 'sqoop eval' 13 | 14 | 15 | DESCRIPTION 16 | ----------- 17 | 18 | include::../user/create-hive-table-purpose.txt[] 19 | 20 | OPTIONS 21 | ------- 22 | 23 | The +--connect+ and +--query+ options are required. 24 | 25 | include::common-args.txt[] 26 | 27 | SQL evaluation options 28 | ~~~~~~~~~~~~~~~~~~~~~~ 29 | 30 | -e:: 31 | --query (statement):: 32 | Execute +statement+ in SQL. 33 | 34 | 35 | ENVIRONMENT 36 | ----------- 37 | 38 | See 'sqoop(1)' 39 | 40 | 41 | //// 42 | Copyright 2011 The Apache Software Foundation 43 | 44 | Licensed to the Apache Software Foundation (ASF) under one 45 | or more contributor license agreements. See the NOTICE file 46 | distributed with this work for additional information 47 | regarding copyright ownership. The ASF licenses this file 48 | to you under the Apache License, Version 2.0 (the 49 | "License"); you may not use this file except in compliance 50 | with the License. You may obtain a copy of the License at 51 | 52 | http://www.apache.org/licenses/LICENSE-2.0 53 | 54 | Unless required by applicable law or agreed to in writing, software 55 | distributed under the License is distributed on an "AS IS" BASIS, 56 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 57 | See the License for the specific language governing permissions and 58 | limitations under the License. 59 | //// 60 | 61 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/util/ExportException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.util; 20 | 21 | /** 22 | * General error during export process. 23 | */ 24 | @SuppressWarnings("serial") 25 | public class ExportException extends Exception { 26 | 27 | public ExportException() { 28 | super("ExportException"); 29 | } 30 | 31 | public ExportException(final String message) { 32 | super(message); 33 | } 34 | 35 | public ExportException(final Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | public ExportException(final String message, final Throwable cause) { 40 | super(message, cause); 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | String msg = getMessage(); 46 | return (null == msg) ? "ExportException" : msg; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/util/ImportException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.util; 20 | 21 | /** 22 | * General error during the import process. 23 | */ 24 | @SuppressWarnings("serial") 25 | public class ImportException extends Exception { 26 | 27 | public ImportException() { 28 | super("ImportException"); 29 | } 30 | 31 | public ImportException(final String message) { 32 | super(message); 33 | } 34 | 35 | public ImportException(final Throwable cause) { 36 | super(cause); 37 | } 38 | 39 | public ImportException(final String message, final Throwable cause) { 40 | super(message, cause); 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | String msg = getMessage(); 46 | return (null == msg) ? "ImportException" : msg; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/docs/man/codegen-args.txt: -------------------------------------------------------------------------------- 1 | 2 | //// 3 | Licensed to the Apache Software Foundation (ASF) under one 4 | or more contributor license agreements. See the NOTICE file 5 | distributed with this work for additional information 6 | regarding copyright ownership. The ASF licenses this file 7 | to you under the Apache License, Version 2.0 (the 8 | "License"); you may not use this file except in compliance 9 | with the License. You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | //// 19 | 20 | Code generation options 21 | ~~~~~~~~~~~~~~~~~~~~~~~ 22 | 23 | --bindir (dir):: 24 | Output directory for compiled objects 25 | 26 | --class-name (name):: 27 | Sets the name of the class to generate. By default, classes are 28 | named after the table they represent. Using this parameters 29 | ignores +--package-name+. 30 | 31 | --jar-file (filename):: 32 | Disable code generation; use specified jar 33 | 34 | --outdir (dir):: 35 | Output directory for generated code 36 | 37 | --delete-compile-dir:: 38 | Remove temporarily generated class Jar files after job finishes 39 | 40 | --package-name (package):: 41 | Puts auto-generated classes in the named Java package 42 | 43 | --map-column-java (mapping):: 44 | Override default mapping from SQL type to Java type for configured columns 45 | -------------------------------------------------------------------------------- /src/java/org/apache/sqoop/tool/JDBCTransactionLevels.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package org.apache.sqoop.tool; 20 | 21 | import java.sql.Connection; 22 | 23 | enum JDBCTransactionLevels { 24 | TRANSACTION_NONE(Connection.TRANSACTION_NONE), 25 | TRANSACTION_READ_UNCOMMITTED(Connection.TRANSACTION_READ_UNCOMMITTED), 26 | TRANSACTION_READ_COMMITTED(Connection.TRANSACTION_READ_COMMITTED), 27 | TRANSACTION_REPEATABLE_READ(Connection.TRANSACTION_REPEATABLE_READ), 28 | TRANSACTION_SERIALIZABLE(Connection.TRANSACTION_SERIALIZABLE); 29 | 30 | private final int transactionLevelValue; 31 | 32 | private JDBCTransactionLevels(int transactionLevelValue) { 33 | this.transactionLevelValue = transactionLevelValue; 34 | } 35 | 36 | public int getTransactionIsolationLevelValue() { 37 | return transactionLevelValue; 38 | } 39 | } 40 | --------------------------------------------------------------------------------