├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ └── check.yml ├── .gitignore ├── CHANGELOG.md ├── COPYING ├── README.md ├── build.gradle ├── ci ├── mysql.yml ├── postgresql.yml ├── redshift.yml └── sqlserver.yml ├── config └── checkstyle │ ├── checkstyle.xml │ └── default.xml ├── embulk-input-jdbc ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ └── main │ └── java │ └── org │ └── embulk │ └── input │ └── jdbc │ ├── AbstractJdbcInputPlugin.java │ ├── JdbcColumn.java │ ├── JdbcColumnOption.java │ ├── JdbcInputConnection.java │ ├── JdbcInputPlugin.java │ ├── JdbcLiteral.java │ ├── JdbcSchema.java │ ├── Ssl.java │ ├── ToString.java │ ├── ToStringMap.java │ └── getter │ ├── AbstractColumnGetter.java │ ├── AbstractIncrementalHandler.java │ ├── AbstractTimestampColumnGetter.java │ ├── BigDecimalColumnGetter.java │ ├── BooleanColumnGetter.java │ ├── ColumnGetter.java │ ├── ColumnGetterFactory.java │ ├── DateColumnGetter.java │ ├── DoubleColumnGetter.java │ ├── FloatColumnGetter.java │ ├── JsonColumnGetter.java │ ├── LongColumnGetter.java │ ├── StringColumnGetter.java │ ├── TimeColumnGetter.java │ ├── TimestampColumnGetter.java │ ├── TimestampWithTimeZoneIncrementalHandler.java │ └── TimestampWithoutTimeZoneIncrementalHandler.java ├── embulk-input-mysql ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── input │ │ └── mysql │ │ ├── MySQLInputConnection.java │ │ ├── MySQLInputPlugin.java │ │ └── getter │ │ ├── AbstractMySQLTimestampIncrementalHandler.java │ │ ├── MySQLColumnGetterFactory.java │ │ ├── MySQLDateTimeTimestampIncrementalHandler.java │ │ └── MySQLTimestampTimestampIncrementalHandler.java │ └── test │ ├── java │ └── org │ │ └── embulk │ │ └── input │ │ └── mysql │ │ ├── BasicTest.java │ │ ├── IncrementalTest.java │ │ ├── MySQLColumnGetterFactoryTest.java │ │ ├── MySQLTests.java │ │ └── OptionTest.java │ └── resources │ └── org │ └── embulk │ └── input │ └── mysql │ └── test │ └── expect │ ├── basic │ ├── setup.sql │ ├── test_boolean_config.yml │ ├── test_boolean_expected.csv │ ├── test_boolean_expected.diff │ ├── test_config.yml │ ├── test_double_config.yml │ ├── test_double_expected.csv │ ├── test_double_expected.diff │ ├── test_expected.csv │ ├── test_expected.diff │ ├── test_invalid_zone_config.yml │ ├── test_long_config.yml │ ├── test_long_expected.csv │ ├── test_long_expected.diff │ ├── test_string_config.yml │ ├── test_string_expected.csv │ ├── test_string_expected.diff │ ├── test_timestamp1_config.yml │ ├── test_timestamp1_expected.csv │ ├── test_timestamp1_expected.diff │ ├── test_timestamp2_config.yml │ ├── test_timestamp2_expected.csv │ ├── test_timestamp2_expected.diff │ ├── test_timestamp3_config.yml │ ├── test_timestamp3_expected.csv │ ├── test_timestamp3_expected.diff │ ├── test_valuetype_decimal_config.yml │ ├── test_valuetype_decimal_expected.csv │ ├── test_valuetype_decimal_expected.diff │ ├── test_valuetype_json_config.yml │ ├── test_valuetype_json_expected.csv │ ├── test_valuetype_json_expected.diff │ ├── test_valuetype_string_config.yml │ ├── test_valuetype_string_expected.csv │ └── test_valuetype_string_expected.diff │ ├── incremental │ ├── char │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ └── setup.sql │ ├── dt │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ └── setup.sql │ ├── int │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ └── setup.sql │ ├── query │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ ├── multi_columns_config_1.yml │ │ ├── multi_columns_config_2.yml │ │ ├── multi_columns_expected_1.csv │ │ ├── multi_columns_expected_1.diff │ │ ├── multi_columns_expected_2.csv │ │ ├── multi_columns_expected_2.diff │ │ ├── same_length_placeholders_config_1.yml │ │ ├── same_length_placeholders_config_2.yml │ │ └── setup.sql │ └── ts │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ └── setup.sql │ └── option │ ├── after_select.yml │ ├── after_select_expected.csv │ ├── before_select.yml │ ├── before_select_expected.csv │ ├── before_setup.yml │ ├── before_setup_expected.csv │ ├── column_options.yml │ ├── column_options_expected.csv │ ├── default_column_options.yml │ ├── default_column_options_expected.csv │ ├── expected1.csv │ ├── input.yml │ ├── order_by.yml │ ├── order_by_asc.yml │ ├── order_by_asc_expected.csv │ ├── order_by_desc.yml │ ├── order_by_desc_expected.csv │ ├── order_by_expected.csv │ └── setup.sql ├── embulk-input-postgresql ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── input │ │ └── postgresql │ │ ├── PostgreSQLInputConnection.java │ │ ├── PostgreSQLInputPlugin.java │ │ └── getter │ │ ├── ArrayColumnGetter.java │ │ ├── HstoreToJsonColumnGetter.java │ │ └── PostgreSQLColumnGetterFactory.java │ └── test │ ├── java │ └── org │ │ └── embulk │ │ └── input │ │ └── postgresql │ │ ├── ArrayTest.java │ │ ├── HstoreTest.java │ │ ├── IncrementalTest.java │ │ ├── PostgreSQLTests.java │ │ └── UUIDTest.java │ └── resources │ └── org │ └── embulk │ └── input │ └── postgresql │ └── test │ └── expect │ ├── array │ ├── as_json.yml │ ├── as_string.yml │ ├── expected_json.csv │ ├── expected_string.csv │ └── setup.sql │ ├── hstore │ ├── as_json.yml │ ├── as_string.yml │ ├── expected_json.csv │ ├── expected_string.csv │ └── setup.sql │ ├── incremental │ ├── char │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ └── setup.sql │ ├── int │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ └── setup.sql │ ├── query │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ ├── multi_columns_config_1.yml │ │ ├── multi_columns_config_2.yml │ │ ├── multi_columns_expected_1.csv │ │ ├── multi_columns_expected_1.diff │ │ ├── multi_columns_expected_2.csv │ │ ├── multi_columns_expected_2.diff │ │ └── setup.sql │ ├── timestamp │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ └── setup.sql │ └── timestamptz │ │ ├── config_1.yml │ │ ├── config_2.yml │ │ ├── expected_1.csv │ │ ├── expected_1.diff │ │ ├── expected_2.csv │ │ ├── expected_2.diff │ │ ├── insert_more.sql │ │ └── setup.sql │ └── uuid │ ├── as_string.yml │ ├── expected_string.csv │ └── setup.sql ├── embulk-input-redshift ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── input │ │ └── redshift │ │ ├── RedshiftInputPlugin.java │ │ └── getter │ │ └── RedshiftColumnGetterFactory.java │ └── test │ ├── java │ └── org │ │ └── embulk │ │ └── input │ │ └── redshift │ │ ├── IncrementalTest.java │ │ └── RedshiftTests.java │ └── resources │ └── org │ └── embulk │ └── input │ └── redshift │ └── test │ └── expect │ └── incremental │ ├── char │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ └── setup.sql │ ├── int │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ └── setup.sql │ ├── query │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ ├── multi_columns_config_1.yml │ ├── multi_columns_config_2.yml │ ├── multi_columns_expected_1.csv │ ├── multi_columns_expected_1.diff │ ├── multi_columns_expected_2.csv │ ├── multi_columns_expected_2.diff │ └── setup.sql │ ├── timestamp │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ └── setup.sql │ └── timestamptz │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ └── setup.sql ├── embulk-input-sqlserver ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── input │ │ └── sqlserver │ │ ├── SQLServerInputConnection.java │ │ ├── SQLServerInputPlugin.java │ │ └── getter │ │ └── SQLServerColumnGetterFactory.java │ └── test │ ├── java │ └── org │ │ └── embulk │ │ └── input │ │ └── sqlserver │ │ ├── BasicTest.java │ │ ├── IncrementalTest.java │ │ └── SQLServerTests.java │ └── resources │ └── org │ └── embulk │ └── input │ └── sqlserver │ └── test │ └── expect │ ├── basic │ ├── jtds_config.yml │ ├── setup.sql │ ├── test_config.yml │ ├── test_expected.csv │ └── test_expected.diff │ └── incremental │ ├── char │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ └── setup.sql │ ├── datetime2 │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ └── setup.sql │ ├── int │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ └── setup.sql │ └── query │ ├── config_1.yml │ ├── config_2.yml │ ├── expected_1.csv │ ├── expected_1.diff │ ├── expected_2.csv │ ├── expected_2.diff │ ├── insert_more.sql │ ├── multi_columns_config_1.yml │ ├── multi_columns_config_2.yml │ ├── multi_columns_expected_1.csv │ ├── multi_columns_expected_1.diff │ ├── multi_columns_expected_2.csv │ ├── multi_columns_expected_2.diff │ └── setup.sql ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── test-scripts ├── mysql-input ├── data │ └── test_expected.csv ├── test.bat ├── test.sql └── test.yml ├── postgresql-input ├── data │ └── test_expected.csv ├── test.bat ├── test.sql └── test.yml ├── sqlserver-input ├── data │ └── test_expected.csv ├── test-jtds.bat ├── test-jtds.yml ├── test.bat ├── test.sql └── test.yml └── test-input.bat /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @embulk/jdbc-maintainers 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | --- 2 | version: 2 3 | updates: 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "daily" 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | /tmp/ 3 | .gradle/ 4 | build/ 5 | .idea 6 | /out/ 7 | ci/postgresql.yml 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | Look at its Git annotated tags, or its [GitHub Releases](https://github.com/embulk/embulk-input-jdbc/releases). 4 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (C) 2015 Sadayuki Furuhashi 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JDBC input plugins for Embulk 2 | 3 | JDBC input plugins for Embulk loads records to databases using JDBC drivers. 4 | 5 | **[WARNING!]** 6 | 7 | - embulk-input-jdbc 0.10 requires Java 8 or later (no longer supports Java 7), and requires embulk 0.9 or later. 8 | - The default embulk type for MySQL JSON type was changed from `string` to `json` since embulk-input-jdbc 0.10 . 9 | 10 | ## MySQL 11 | 12 | See [embulk-input-mysql](embulk-input-mysql/). 13 | 14 | ## PostgreSQL 15 | 16 | See [embulk-input-postgresql](embulk-input-postgresql/). 17 | 18 | ## Redshift 19 | 20 | See [embulk-input-redshift](embulk-input-redshift/). 21 | 22 | ## SQL Server 23 | 24 | See [embulk-input-sqlserver](embulk-input-sqlserver/). 25 | 26 | ## Others (generic JDBC) 27 | 28 | See [embulk-input-jdbc](embulk-input-jdbc/). 29 | -------------------------------------------------------------------------------- /ci/mysql.yml: -------------------------------------------------------------------------------- 1 | type: mysql 2 | host: 127.0.0.1 3 | port: 3306 4 | database: ci_test 5 | user: root 6 | password: root 7 | -------------------------------------------------------------------------------- /ci/postgresql.yml: -------------------------------------------------------------------------------- 1 | type: postgresql 2 | host: 127.0.0.1 3 | port: 5432 4 | database: ci_test 5 | user: postgres 6 | password: postgres 7 | -------------------------------------------------------------------------------- /ci/redshift.yml: -------------------------------------------------------------------------------- 1 | type: redshift 2 | host: 127.0.0.1 3 | port: 5439 4 | database: ci_test 5 | user: postgres 6 | password: postgres 7 | fetch_rows: 1000 8 | -------------------------------------------------------------------------------- /ci/sqlserver.yml: -------------------------------------------------------------------------------- 1 | type: sqlserver 2 | host: 127.0.0.1 3 | port: 1433 4 | database: testdb 5 | user: sa 6 | password: "P@ssw0rd" 7 | -------------------------------------------------------------------------------- /embulk-input-jdbc/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | } 3 | 4 | embulkPlugin { 5 | mainClass = "org.embulk.input.jdbc.JdbcInputPlugin" 6 | category = "input" 7 | type = "jdbc" 8 | } 9 | 10 | publishing { 11 | publications { 12 | maven(MavenPublication) { 13 | pom { // https://central.sonatype.org/pages/requirements.html 14 | developers { 15 | developer { 16 | name = "Sadayuki Furuhashi" 17 | email = "frsyuki@gmail.com" 18 | } 19 | developer { 20 | name = "Hitoshi Tanaka" 21 | email = "thitoshi@cac.co.jp" 22 | } 23 | developer { 24 | name = "Tomohiro Hashidate" 25 | email = "kakyoin.hierophant@gmail.com" 26 | } 27 | developer { 28 | name = "You Yamagata" 29 | email = "youy.bg8@gmail.com" 30 | } 31 | developer { 32 | name = "Shinichi Ishimura" 33 | email = "shiketaudonko41@gmail.com" 34 | } 35 | developer { 36 | name = "Satoshi Akama" 37 | email = "satoshiakama@gmail.com" 38 | } 39 | developer { 40 | name = "Muga Nishizawa" 41 | email = "muga.nishizawa@gmail.com" 42 | } 43 | developer { 44 | name = "Hiroyuki Sato" 45 | email = "hiroysato@gmail.com" 46 | } 47 | developer { 48 | name = "Dai MIKURUBE" 49 | email = "dmikurube@treasure-data.com" 50 | } 51 | } 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /embulk-input-jdbc/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 9 | org.embulk:embulk-spi:0.11=compileClasspath 10 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 11 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 12 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 13 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 14 | org.msgpack:msgpack-core:0.8.24=compileClasspath 15 | org.slf4j:slf4j-api:2.0.7=compileClasspath 16 | empty= 17 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcColumn.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | 6 | public class JdbcColumn 7 | { 8 | private String name; 9 | private String typeName; 10 | private int sqlType; 11 | private int precision; 12 | private int scale; 13 | 14 | @JsonCreator 15 | public JdbcColumn( 16 | @JsonProperty("name") String name, 17 | @JsonProperty("typeName") String typeName, 18 | @JsonProperty("sqlType") int sqlType, 19 | @JsonProperty("precision") int precision, 20 | @JsonProperty("scale") int scale) 21 | { 22 | this.name = name; 23 | this.typeName = typeName; 24 | this.sqlType = sqlType; 25 | this.precision = precision; 26 | this.scale = scale; 27 | } 28 | 29 | @JsonProperty("name") 30 | public String getName() 31 | { 32 | return name; 33 | } 34 | 35 | @JsonProperty("typeName") 36 | public String getTypeName() 37 | { 38 | return typeName; 39 | } 40 | 41 | @JsonProperty("sqlType") 42 | public int getSqlType() 43 | { 44 | return sqlType; 45 | } 46 | 47 | @JsonProperty("precision") 48 | public int getPrecision() 49 | { 50 | return precision; 51 | } 52 | 53 | @JsonProperty("scale") 54 | public int getScale() 55 | { 56 | return scale; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcColumnOption.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc; 2 | 3 | import java.time.ZoneId; 4 | import java.util.Optional; 5 | import org.embulk.spi.type.Type; 6 | import org.embulk.util.config.Config; 7 | import org.embulk.util.config.ConfigDefault; 8 | import org.embulk.util.config.Task; 9 | 10 | public interface JdbcColumnOption 11 | extends Task 12 | { 13 | @Config("value_type") 14 | @ConfigDefault("\"coalesce\"") 15 | public String getValueType(); 16 | 17 | @Config("type") 18 | @ConfigDefault("null") 19 | public Optional getType(); 20 | 21 | @Config("timestamp_format") 22 | @ConfigDefault("null") 23 | public Optional getTimestampFormat(); 24 | 25 | @Config("timezone") 26 | @ConfigDefault("null") 27 | public Optional getTimeZone(); 28 | } 29 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcInputPlugin.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc; 2 | 3 | import java.util.Set; 4 | import java.util.HashSet; 5 | import java.util.Optional; 6 | import java.util.Properties; 7 | import java.sql.Connection; 8 | import java.sql.Driver; 9 | import java.sql.SQLException; 10 | import org.embulk.util.config.Config; 11 | import org.embulk.util.config.ConfigDefault; 12 | 13 | public class JdbcInputPlugin 14 | extends AbstractJdbcInputPlugin 15 | { 16 | private final static Set loadedJarGlobs = new HashSet(); 17 | 18 | public interface GenericPluginTask 19 | extends PluginTask 20 | { 21 | @Config("driver_path") 22 | @ConfigDefault("null") 23 | public Optional getDriverPath(); 24 | 25 | @Config("driver_class") 26 | public String getDriverClass(); 27 | 28 | @Config("url") 29 | public String getUrl(); 30 | 31 | @Config("user") 32 | @ConfigDefault("null") 33 | public Optional getUser(); 34 | 35 | @Config("password") 36 | @ConfigDefault("null") 37 | public Optional getPassword(); 38 | 39 | @Config("schema") 40 | @ConfigDefault("null") 41 | public Optional getSchema(); 42 | } 43 | 44 | @Override 45 | protected Class getTaskClass() 46 | { 47 | return GenericPluginTask.class; 48 | } 49 | 50 | @Override 51 | protected JdbcInputConnection newConnection(PluginTask task) throws SQLException 52 | { 53 | GenericPluginTask t = (GenericPluginTask) task; 54 | 55 | if (t.getDriverPath().isPresent()) { 56 | synchronized (loadedJarGlobs) { 57 | String glob = t.getDriverPath().get(); 58 | if (!loadedJarGlobs.contains(glob)) { 59 | addDriverJarToClasspath(glob); 60 | loadedJarGlobs.add(glob); 61 | } 62 | } 63 | } 64 | 65 | Properties props = new Properties(); 66 | if (t.getUser().isPresent()) { 67 | props.setProperty("user", t.getUser().get()); 68 | } 69 | if (t.getPassword().isPresent()) { 70 | props.setProperty("password", t.getPassword().get()); 71 | } 72 | 73 | props.putAll(t.getOptions()); 74 | logConnectionProperties(t.getUrl(), props); 75 | 76 | Driver driver; 77 | try { 78 | // TODO check Class.forName(driverClass) is a Driver before newInstance 79 | // for security 80 | driver = (Driver) Class.forName(t.getDriverClass()).newInstance(); 81 | } catch (final RuntimeException ex) { 82 | throw ex; 83 | } catch (final Exception ex) { 84 | throw new RuntimeException(ex); 85 | } 86 | 87 | Connection con = driver.connect(t.getUrl(), props); 88 | try { 89 | JdbcInputConnection c = new JdbcInputConnection(con, t.getSchema().orElse(null)); 90 | con = null; 91 | return c; 92 | } finally { 93 | if (con != null) { 94 | con.close(); 95 | } 96 | } 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcLiteral.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonProperty; 5 | import com.fasterxml.jackson.databind.JsonNode; 6 | 7 | public class JdbcLiteral 8 | { 9 | private final int columnIndex; 10 | private final JsonNode value; 11 | 12 | @JsonCreator 13 | public JdbcLiteral( 14 | @JsonProperty("columnIndex") int columnIndex, 15 | @JsonProperty("value") JsonNode value) 16 | { 17 | this.columnIndex = columnIndex; 18 | this.value = value; 19 | } 20 | 21 | @JsonProperty("columnIndex") 22 | public int getColumnIndex() 23 | { 24 | return columnIndex; 25 | } 26 | 27 | @JsonProperty("value") 28 | public JsonNode getValue() 29 | { 30 | return value; 31 | } 32 | 33 | @Override 34 | public String toString() 35 | { 36 | return value.toString(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/JdbcSchema.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc; 2 | 3 | import java.util.List; 4 | import java.util.Optional; 5 | import com.fasterxml.jackson.annotation.JsonCreator; 6 | import com.fasterxml.jackson.annotation.JsonValue; 7 | 8 | public class JdbcSchema 9 | { 10 | private List columns; 11 | 12 | @JsonCreator 13 | public JdbcSchema(List columns) 14 | { 15 | this.columns = columns; 16 | } 17 | 18 | @JsonValue 19 | public List getColumns() 20 | { 21 | return columns; 22 | } 23 | 24 | public int getCount() 25 | { 26 | return columns.size(); 27 | } 28 | 29 | public JdbcColumn getColumn(int i) 30 | { 31 | return columns.get(i); 32 | } 33 | 34 | public String getColumnName(int i) 35 | { 36 | return columns.get(i).getName(); 37 | } 38 | 39 | public Optional findColumn(String caseInsensitiveName) 40 | { 41 | // find by case sensitive first 42 | for (int i = 0; i < columns.size(); i++) { 43 | if (getColumn(i).getName().equals(caseInsensitiveName)) { 44 | return Optional.of(i); 45 | } 46 | } 47 | // find by case insensitive 48 | for (int i = 0; i < columns.size(); i++) { 49 | if (getColumn(i).getName().equalsIgnoreCase(caseInsensitiveName)) { 50 | return Optional.of(i); 51 | } 52 | } 53 | return Optional.empty(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/Ssl.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonValue; 5 | 6 | import org.embulk.config.ConfigException; 7 | 8 | public enum Ssl 9 | { 10 | ENABLE, 11 | DISABLE, 12 | VERIFY; 13 | 14 | @JsonValue 15 | @Override 16 | public String toString() 17 | { 18 | return this.name().toLowerCase(); 19 | } 20 | 21 | @JsonCreator 22 | public static Ssl fromString(String value) 23 | { 24 | switch(value) { 25 | case "enable": 26 | case "true": 27 | return ENABLE; 28 | case "disable": 29 | case "false": 30 | return DISABLE; 31 | case "verify": 32 | return VERIFY; 33 | default: 34 | throw new ConfigException(String.format("Unknown SSL value '%s'. Supported values are enable, true, disable, false or verify.", value)); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/ToString.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.JsonMappingException; 5 | import com.fasterxml.jackson.databind.node.NullNode; 6 | import com.fasterxml.jackson.annotation.JsonCreator; 7 | import com.fasterxml.jackson.annotation.JsonValue; 8 | import java.util.Optional; 9 | 10 | public class ToString 11 | { 12 | private final String string; 13 | 14 | public ToString(String string) 15 | { 16 | this.string = string; 17 | } 18 | 19 | @JsonCreator 20 | ToString(Optional option) throws JsonMappingException 21 | { 22 | JsonNode node = option.orElse(NullNode.getInstance()); 23 | if (node.isTextual()) { 24 | this.string = node.textValue(); 25 | } else if (node.isValueNode()) { 26 | this.string = node.toString(); 27 | } else { 28 | throw new JsonMappingException(String.format("Arrays and objects are invalid: '%s'", node)); 29 | } 30 | } 31 | 32 | @Override 33 | public boolean equals(Object obj) 34 | { 35 | if (!(obj instanceof ToString)) { 36 | return false; 37 | } 38 | ToString o = (ToString) obj; 39 | return string.equals(o.string); 40 | } 41 | 42 | @Override 43 | public int hashCode() 44 | { 45 | return string.hashCode(); 46 | } 47 | 48 | @JsonValue 49 | @Override 50 | public String toString() 51 | { 52 | return string; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/ToStringMap.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc; 2 | 3 | import java.util.Map; 4 | import java.util.HashMap; 5 | import java.util.Properties; 6 | import com.fasterxml.jackson.annotation.JsonCreator; 7 | 8 | public class ToStringMap 9 | extends HashMap 10 | { 11 | @JsonCreator 12 | ToStringMap(Map map) 13 | { 14 | super(mapToStringString(map)); 15 | } 16 | 17 | public Properties toProperties() 18 | { 19 | Properties props = new Properties(); 20 | props.putAll(this); 21 | return props; 22 | } 23 | 24 | private static Map mapToStringString(final Map mapOfToString) { 25 | final HashMap result = new HashMap<>(); 26 | for (final Map.Entry entry : mapOfToString.entrySet()) { 27 | final ToString value = entry.getValue(); 28 | if (value == null) { 29 | result.put(entry.getKey(), "null"); 30 | } else { 31 | result.put(entry.getKey(), value.toString()); 32 | } 33 | } 34 | return result; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | import com.fasterxml.jackson.databind.JsonNode; 7 | import com.fasterxml.jackson.databind.node.JsonNodeFactory; 8 | import org.embulk.spi.Column; 9 | import org.embulk.spi.ColumnVisitor; 10 | import org.embulk.spi.PageBuilder; 11 | import org.embulk.spi.type.Type; 12 | import org.embulk.spi.DataException; 13 | import static java.util.Locale.ENGLISH; 14 | 15 | public abstract class AbstractIncrementalHandler implements ColumnGetter 16 | { 17 | protected static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance; 18 | 19 | protected ColumnGetter next; 20 | 21 | public AbstractIncrementalHandler(ColumnGetter next) 22 | { 23 | this.next = next; 24 | } 25 | 26 | @Override 27 | public void getAndSet(ResultSet from, int fromIndex, 28 | Column toColumn) throws SQLException 29 | { 30 | next.getAndSet(from, fromIndex, toColumn); 31 | } 32 | 33 | @Override 34 | public Type getToType() 35 | { 36 | return next.getToType(); 37 | } 38 | 39 | @Override 40 | public abstract JsonNode encodeToJson(); 41 | 42 | @Override 43 | public abstract void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue) 44 | throws SQLException; 45 | } 46 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.time.Instant; 4 | import org.embulk.spi.Column; 5 | import org.embulk.spi.PageBuilder; 6 | import org.embulk.spi.type.Type; 7 | import org.embulk.util.timestamp.TimestampFormatter; 8 | 9 | public abstract class AbstractTimestampColumnGetter 10 | extends AbstractColumnGetter 11 | { 12 | protected final TimestampFormatter timestampFormatter; 13 | protected Instant value; 14 | 15 | public AbstractTimestampColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter) 16 | { 17 | super(to, toType); 18 | 19 | this.timestampFormatter = timestampFormatter; 20 | } 21 | 22 | @Override 23 | public void stringColumn(Column column) 24 | { 25 | to.setString(column, timestampFormatter.format(value)); 26 | } 27 | 28 | @Override 29 | public void jsonColumn(Column column) { 30 | throw new UnsupportedOperationException("This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type."); 31 | } 32 | 33 | @Override 34 | public void timestampColumn(Column column) 35 | { 36 | to.setTimestamp(column, org.embulk.spi.time.Timestamp.ofInstant(value)); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.math.BigDecimal; 4 | import java.sql.ResultSet; 5 | import java.sql.SQLException; 6 | import org.embulk.spi.Column; 7 | import org.embulk.spi.PageBuilder; 8 | import org.embulk.spi.type.Type; 9 | import org.embulk.spi.type.Types; 10 | 11 | public class BigDecimalColumnGetter 12 | extends AbstractColumnGetter 13 | { 14 | protected BigDecimal value; 15 | 16 | public BigDecimalColumnGetter(PageBuilder to, Type toType) 17 | { 18 | super(to, toType); 19 | } 20 | 21 | @Override 22 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 23 | { 24 | value = from.getBigDecimal(fromIndex); 25 | } 26 | 27 | @Override 28 | protected Type getDefaultToType() 29 | { 30 | return Types.DOUBLE; 31 | } 32 | 33 | @Override 34 | public void booleanColumn(Column column) 35 | { 36 | to.setBoolean(column, value.signum() > 0); 37 | } 38 | 39 | @Override 40 | public void longColumn(Column column) 41 | { 42 | to.setLong(column, value.longValue()); 43 | } 44 | 45 | @Override 46 | public void doubleColumn(Column column) 47 | { 48 | // rounded value could be Double.NEGATIVE_INFINITY or Double.POSITIVE_INFINITY. 49 | double rounded = value.doubleValue(); 50 | to.setDouble(column, rounded); 51 | } 52 | 53 | @Override 54 | public void stringColumn(Column column) 55 | { 56 | to.setString(column, value.toPlainString()); 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | import org.embulk.spi.Column; 6 | import org.embulk.spi.PageBuilder; 7 | import org.embulk.spi.type.Type; 8 | import org.embulk.spi.type.Types; 9 | 10 | public class BooleanColumnGetter 11 | extends AbstractColumnGetter 12 | { 13 | protected boolean value; 14 | 15 | public BooleanColumnGetter(PageBuilder to, Type toType) 16 | { 17 | super(to, toType); 18 | } 19 | 20 | @Override 21 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 22 | { 23 | value = from.getBoolean(fromIndex); 24 | } 25 | 26 | @Override 27 | protected Type getDefaultToType() 28 | { 29 | return Types.BOOLEAN; 30 | } 31 | 32 | @Override 33 | public void booleanColumn(Column column) 34 | { 35 | to.setBoolean(column, value); 36 | } 37 | 38 | @Override 39 | public void longColumn(Column column) 40 | { 41 | to.setLong(column, value ? 1L : 0L); 42 | } 43 | 44 | @Override 45 | public void doubleColumn(Column column) 46 | { 47 | to.setDouble(column, value ? 1.0 : 0.0); 48 | } 49 | 50 | @Override 51 | public void stringColumn(Column column) 52 | { 53 | to.setString(column, Boolean.toString(value)); 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/ColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | import java.sql.PreparedStatement; 6 | import com.fasterxml.jackson.databind.JsonNode; 7 | import org.embulk.spi.Column; 8 | import org.embulk.spi.type.Type; 9 | 10 | public interface ColumnGetter 11 | { 12 | public void getAndSet(ResultSet from, int fromIndex, 13 | Column toColumn) throws SQLException; 14 | 15 | public Type getToType(); 16 | 17 | public JsonNode encodeToJson(); 18 | 19 | public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue) 20 | throws SQLException; 21 | } 22 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/DateColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.Date; 4 | import java.sql.ResultSet; 5 | import java.sql.SQLException; 6 | import java.time.Instant; 7 | import org.embulk.spi.PageBuilder; 8 | import org.embulk.spi.type.Type; 9 | import org.embulk.spi.type.Types; 10 | import org.embulk.util.timestamp.TimestampFormatter; 11 | 12 | public class DateColumnGetter 13 | extends AbstractTimestampColumnGetter 14 | { 15 | static final String DEFAULT_FORMAT = "%Y-%m-%d"; 16 | 17 | public DateColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter) 18 | { 19 | super(to, toType, timestampFormatter); 20 | } 21 | 22 | @Override 23 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 24 | { 25 | Date date = from.getDate(fromIndex); 26 | if (date != null) { 27 | value = Instant.ofEpochMilli(date.getTime()); 28 | } 29 | } 30 | 31 | @Override 32 | protected Type getDefaultToType() 33 | { 34 | return Types.TIMESTAMP.withFormat(DEFAULT_FORMAT); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/DoubleColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.math.RoundingMode; 4 | import java.sql.ResultSet; 5 | import java.sql.SQLException; 6 | import org.embulk.spi.Column; 7 | import org.embulk.spi.PageBuilder; 8 | import org.embulk.spi.type.Type; 9 | import org.embulk.spi.type.Types; 10 | 11 | public class DoubleColumnGetter 12 | extends AbstractColumnGetter 13 | { 14 | protected double value; 15 | 16 | public DoubleColumnGetter(PageBuilder to, Type toType) 17 | { 18 | super(to, toType); 19 | } 20 | 21 | @Override 22 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 23 | { 24 | value = from.getDouble(fromIndex); 25 | } 26 | 27 | @Override 28 | protected Type getDefaultToType() 29 | { 30 | return Types.DOUBLE; 31 | } 32 | 33 | @Override 34 | public void booleanColumn(Column column) 35 | { 36 | to.setBoolean(column, value > 0.0); 37 | } 38 | 39 | @Override 40 | public void longColumn(Column column) 41 | { 42 | long l; 43 | try { 44 | l = roundDoubleToLong(this.value); 45 | } catch (ArithmeticException e) { 46 | // NaN / Infinite / -Infinite 47 | super.longColumn(column); 48 | return; 49 | } 50 | to.setLong(column, l); 51 | } 52 | 53 | @Override 54 | public void doubleColumn(Column column) 55 | { 56 | to.setDouble(column, value); 57 | } 58 | 59 | @Override 60 | public void stringColumn(Column column) 61 | { 62 | to.setString(column, Double.toString(value)); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/FloatColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.math.RoundingMode; 4 | import java.sql.ResultSet; 5 | import java.sql.SQLException; 6 | import org.embulk.spi.Column; 7 | import org.embulk.spi.PageBuilder; 8 | import org.embulk.spi.type.Type; 9 | import org.embulk.spi.type.Types; 10 | 11 | public class FloatColumnGetter 12 | extends AbstractColumnGetter 13 | { 14 | protected float value; 15 | 16 | public FloatColumnGetter(PageBuilder to, Type toType) 17 | { 18 | super(to, toType); 19 | } 20 | 21 | @Override 22 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 23 | { 24 | value = from.getFloat(fromIndex); 25 | } 26 | 27 | @Override 28 | protected Type getDefaultToType() 29 | { 30 | return Types.DOUBLE; 31 | } 32 | 33 | @Override 34 | public void booleanColumn(Column column) 35 | { 36 | to.setBoolean(column, value > 0.0); 37 | } 38 | 39 | @Override 40 | public void longColumn(Column column) 41 | { 42 | long l; 43 | try { 44 | l = roundFloatToLong(this.value); 45 | } catch (ArithmeticException e) { 46 | // NaN / Infinite / -Infinite 47 | super.longColumn(column); 48 | return; 49 | } 50 | to.setLong(column, l); 51 | } 52 | 53 | @Override 54 | public void doubleColumn(Column column) 55 | { 56 | to.setDouble(column, value); 57 | } 58 | 59 | @Override 60 | public void stringColumn(Column column) 61 | { 62 | to.setString(column, Float.toString(value)); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/JsonColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | 6 | import org.embulk.input.jdbc.getter.AbstractColumnGetter; 7 | import org.embulk.spi.Column; 8 | import org.embulk.spi.PageBuilder; 9 | import org.embulk.spi.type.Type; 10 | import org.embulk.spi.type.Types; 11 | import org.embulk.util.json.JsonParseException; 12 | import org.embulk.util.json.JsonParser; 13 | import org.msgpack.value.Value; 14 | 15 | public class JsonColumnGetter 16 | extends AbstractColumnGetter 17 | { 18 | protected final JsonParser jsonParser = new JsonParser(); 19 | 20 | protected String value; 21 | 22 | public JsonColumnGetter(PageBuilder to, Type toType) 23 | { 24 | super(to, toType); 25 | } 26 | 27 | @Override 28 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 29 | { 30 | value = from.getString(fromIndex); 31 | } 32 | 33 | @Override 34 | protected Type getDefaultToType() 35 | { 36 | return Types.JSON; 37 | } 38 | 39 | @Override 40 | public void jsonColumn(Column column) 41 | { 42 | Value v; 43 | try { 44 | v = jsonParser.parse(value); 45 | } catch (JsonParseException e) { 46 | super.jsonColumn(column); 47 | return; 48 | } 49 | to.setJson(column, v); 50 | } 51 | 52 | @Override 53 | public void stringColumn(Column column) 54 | { 55 | to.setString(column, value); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/LongColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | import com.fasterxml.jackson.databind.JsonNode; 7 | import org.embulk.spi.Column; 8 | import org.embulk.spi.PageBuilder; 9 | import org.embulk.spi.type.Type; 10 | import org.embulk.spi.type.Types; 11 | 12 | public class LongColumnGetter 13 | extends AbstractColumnGetter 14 | { 15 | protected long value; 16 | 17 | public LongColumnGetter(PageBuilder to, Type toType) 18 | { 19 | super(to, toType); 20 | } 21 | 22 | @Override 23 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 24 | { 25 | value = from.getLong(fromIndex); 26 | } 27 | 28 | @Override 29 | protected Type getDefaultToType() 30 | { 31 | return Types.LONG; 32 | } 33 | 34 | @Override 35 | public void booleanColumn(Column column) 36 | { 37 | to.setBoolean(column, value > 0L); 38 | } 39 | 40 | @Override 41 | public void longColumn(Column column) 42 | { 43 | to.setLong(column, value); 44 | } 45 | 46 | @Override 47 | public void doubleColumn(Column column) 48 | { 49 | to.setDouble(column, value); 50 | } 51 | 52 | @Override 53 | public void stringColumn(Column column) 54 | { 55 | to.setString(column, Long.toString(value)); 56 | } 57 | 58 | @Override 59 | public JsonNode encodeToJson() 60 | { 61 | return jsonNodeFactory.numberNode(value); 62 | } 63 | 64 | @Override 65 | public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue) 66 | throws SQLException 67 | { 68 | toStatement.setLong(toIndex, fromValue.asLong()); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.PreparedStatement; 5 | import java.sql.SQLException; 6 | import com.fasterxml.jackson.databind.JsonNode; 7 | import org.embulk.spi.Column; 8 | import org.embulk.spi.PageBuilder; 9 | import org.embulk.spi.type.Type; 10 | import org.embulk.spi.type.Types; 11 | import org.embulk.util.json.JsonParseException; 12 | import org.embulk.util.json.JsonParser; 13 | import org.msgpack.value.Value; 14 | 15 | public class StringColumnGetter 16 | extends AbstractColumnGetter 17 | { 18 | protected final JsonParser jsonParser = new JsonParser(); 19 | 20 | protected String value; 21 | 22 | public StringColumnGetter(PageBuilder to, Type toType) 23 | { 24 | super(to, toType); 25 | } 26 | 27 | @Override 28 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 29 | { 30 | value = from.getString(fromIndex); 31 | } 32 | 33 | @Override 34 | protected Type getDefaultToType() 35 | { 36 | return Types.STRING; 37 | } 38 | 39 | @Override 40 | public void longColumn(Column column) 41 | { 42 | long l; 43 | try { 44 | l = Long.parseLong(value); 45 | } catch (NumberFormatException e) { 46 | super.longColumn(column); 47 | return; 48 | } 49 | to.setLong(column, l); 50 | } 51 | 52 | @Override 53 | public void doubleColumn(Column column) 54 | { 55 | double d; 56 | try { 57 | d = Double.parseDouble(value); 58 | } catch (NumberFormatException e) { 59 | super.doubleColumn(column); 60 | return; 61 | } 62 | to.setDouble(column, d); 63 | } 64 | 65 | @Override 66 | public void jsonColumn(Column column) 67 | { 68 | Value v; 69 | try { 70 | v = jsonParser.parse(value); 71 | } catch (JsonParseException e) { 72 | super.jsonColumn(column); 73 | return; 74 | } 75 | to.setJson(column, v); 76 | } 77 | 78 | @Override 79 | public void stringColumn(Column column) 80 | { 81 | to.setString(column, value); 82 | } 83 | 84 | @Override 85 | public JsonNode encodeToJson() 86 | { 87 | return jsonNodeFactory.textNode(value); 88 | } 89 | 90 | @Override 91 | public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue) 92 | throws SQLException 93 | { 94 | toStatement.setString(toIndex, fromValue.asText()); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | import java.sql.Time; 6 | import java.time.Instant; 7 | import org.embulk.spi.PageBuilder; 8 | import org.embulk.spi.type.Type; 9 | import org.embulk.spi.type.Types; 10 | import org.embulk.util.timestamp.TimestampFormatter; 11 | 12 | public class TimeColumnGetter 13 | extends AbstractTimestampColumnGetter 14 | { 15 | static final String DEFAULT_FORMAT = "%H:%M:%S"; 16 | 17 | public TimeColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter) 18 | { 19 | super(to, toType, timestampFormatter); 20 | } 21 | 22 | @Override 23 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 24 | { 25 | Time time = from.getTime(fromIndex); 26 | if (time != null) { 27 | value = Instant.ofEpochMilli(time.getTime()); 28 | } 29 | } 30 | 31 | @Override 32 | protected Type getDefaultToType() 33 | { 34 | return Types.TIMESTAMP.withFormat(DEFAULT_FORMAT); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.SQLException; 5 | import org.embulk.spi.PageBuilder; 6 | import org.embulk.spi.type.Type; 7 | import org.embulk.spi.type.Types; 8 | import org.embulk.util.timestamp.TimestampFormatter; 9 | 10 | public class TimestampColumnGetter 11 | extends AbstractTimestampColumnGetter 12 | { 13 | static final String DEFAULT_FORMAT = "%Y-%m-%d %H:%M:%S"; 14 | 15 | public TimestampColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter) 16 | { 17 | super(to, toType, timestampFormatter); 18 | } 19 | 20 | @Override 21 | protected void fetch(ResultSet from, int fromIndex) throws SQLException 22 | { 23 | java.sql.Timestamp timestamp = from.getTimestamp(fromIndex); 24 | if (timestamp != null) { 25 | value = timestamp.toInstant(); 26 | } 27 | } 28 | 29 | @Override 30 | protected Type getDefaultToType() 31 | { 32 | return Types.TIMESTAMP.withFormat(DEFAULT_FORMAT); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | 5 | import java.sql.PreparedStatement; 6 | import java.sql.ResultSet; 7 | import java.sql.SQLException; 8 | import java.sql.Timestamp; 9 | import java.time.Instant; 10 | import java.time.format.DateTimeParseException; 11 | import java.util.Optional; 12 | 13 | import org.embulk.config.ConfigException; 14 | import org.embulk.config.ConfigSource; 15 | import org.embulk.spi.Column; 16 | import org.embulk.spi.Exec; 17 | import org.embulk.util.config.Task; 18 | import org.embulk.util.timestamp.TimestampFormatter; 19 | 20 | public class TimestampWithTimeZoneIncrementalHandler 21 | extends AbstractIncrementalHandler 22 | { 23 | // maybe "...%6N%Z", but shouldn't correct for compatibility. 24 | private static final TimestampFormatter FORMATTER = TimestampFormatter.builderWithRuby("%Y-%m-%dT%H:%M:%S.%6NZ").build(); 25 | 26 | private static final TimestampFormatter PARSER = TimestampFormatter.builderWithRuby("%Y-%m-%dT%H:%M:%S.%N%z").build(); 27 | 28 | private long epochSecond; 29 | private int nano; 30 | 31 | public TimestampWithTimeZoneIncrementalHandler(ColumnGetter next) 32 | { 33 | super(next); 34 | } 35 | 36 | @Override 37 | public void getAndSet(ResultSet from, int fromIndex, 38 | Column toColumn) throws SQLException 39 | { 40 | // sniff the value 41 | Timestamp timestamp = from.getTimestamp(fromIndex); 42 | if (timestamp != null) { 43 | epochSecond = timestamp.getTime() / 1000; 44 | nano = timestamp.getNanos(); 45 | } 46 | 47 | super.getAndSet(from, fromIndex, toColumn); 48 | } 49 | 50 | @Override 51 | public JsonNode encodeToJson() 52 | { 53 | return jsonNodeFactory.textNode(format(epochSecond, nano)); 54 | } 55 | 56 | private String format(long epochSecond, int nano) 57 | { 58 | return FORMATTER.format(Instant.ofEpochSecond(epochSecond, nano)); 59 | } 60 | 61 | @Override 62 | public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue) 63 | throws SQLException 64 | { 65 | try { 66 | final Instant epoch = PARSER.parse(fromValue.asText()); 67 | final Timestamp sqlTimestamp = Timestamp.from(epoch); 68 | sqlTimestamp.setNanos(epoch.getNano()); 69 | toStatement.setTimestamp(toIndex, sqlTimestamp); 70 | 71 | } catch (final DateTimeParseException e) { 72 | long now = System.currentTimeMillis(); 73 | String sample = format(now / 1000, (int)((now % 1000)*1000000)); 74 | throw new ConfigException("Invalid timestamp with time zone pattern: " + fromValue + "." 75 | + " The pattern must be 'yyyy-MM-ddTHH:mm:ss.SSSSSSZ'." 76 | + " e.g. \"" + sample + "\" or \"" + sample.replace("Z", "+0000") + "\""); 77 | } 78 | 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.jdbc.getter; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import java.sql.PreparedStatement; 5 | import java.sql.ResultSet; 6 | import java.sql.SQLException; 7 | import java.sql.Timestamp; 8 | import java.time.LocalDateTime; 9 | import java.util.regex.Matcher; 10 | import java.util.regex.Pattern; 11 | import org.embulk.config.ConfigException; 12 | import org.embulk.spi.Column; 13 | import static java.util.Locale.ENGLISH; 14 | 15 | public class TimestampWithoutTimeZoneIncrementalHandler 16 | extends AbstractIncrementalHandler 17 | { 18 | private static final String ISO_USEC_FORMAT = "%d-%02d-%02dT%02d:%02d:%02d.%06d"; 19 | private static final Pattern ISO_USEC_PATTERN = Pattern.compile("(\\d+)-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2}).(\\d{6})"); 20 | 21 | private Timestamp dateTime; 22 | 23 | public TimestampWithoutTimeZoneIncrementalHandler(ColumnGetter next) 24 | { 25 | super(next); 26 | } 27 | 28 | @Override 29 | public void getAndSet(ResultSet from, int fromIndex, 30 | Column toColumn) throws SQLException 31 | { 32 | // sniff the value 33 | Timestamp timestamp = from.getTimestamp(fromIndex); 34 | if (timestamp != null) { 35 | this.dateTime = timestamp; 36 | } 37 | 38 | super.getAndSet(from, fromIndex, toColumn); 39 | } 40 | 41 | @Override 42 | public JsonNode encodeToJson() 43 | { 44 | return jsonNodeFactory.textNode(format(dateTime)); 45 | } 46 | 47 | private String format(Timestamp dateTime) 48 | { 49 | final LocalDateTime localDateTime = dateTime.toLocalDateTime(); 50 | return String.format(ENGLISH, 51 | ISO_USEC_FORMAT, 52 | localDateTime.getYear(), 53 | localDateTime.getMonthValue(), 54 | localDateTime.getDayOfMonth(), 55 | localDateTime.getHour(), 56 | localDateTime.getMinute(), 57 | localDateTime.getSecond(), 58 | localDateTime.getNano() / 1000); 59 | } 60 | 61 | @Override 62 | public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue) 63 | throws SQLException 64 | { 65 | Matcher matcher = ISO_USEC_PATTERN.matcher(fromValue.asText()); 66 | if (!matcher.matches()) { 67 | throw new ConfigException("Invalid timestamp without time zone pattern: " + fromValue + "." 68 | + " The pattern must be 'yyyy-MM-ddTHH:mm:ss.SSSSSS'." 69 | + " e.g. \"" + format(new Timestamp(System.currentTimeMillis())) + "\""); 70 | } 71 | final LocalDateTime localDateTime = LocalDateTime.of( 72 | Integer.parseInt(matcher.group(1)), // year 73 | Integer.parseInt(matcher.group(2)), // month 74 | Integer.parseInt(matcher.group(3)), // day 75 | Integer.parseInt(matcher.group(4)), // hour 76 | Integer.parseInt(matcher.group(5)), // minute 77 | Integer.parseInt(matcher.group(6)), // second 78 | Integer.parseInt(matcher.group(7)) * 1000); // usec -> nsec 79 | final Timestamp sqlDateTime = Timestamp.valueOf(localDateTime); 80 | toStatement.setTimestamp(toIndex, sqlDateTime); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /embulk-input-mysql/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation(project(path: ":embulk-input-jdbc", configuration: "runtimeElements")) 3 | 4 | compileOnly "mysql:mysql-connector-java:5.1.44" 5 | defaultJdbcDriver 'mysql:mysql-connector-java:5.1.44' 6 | 7 | testImplementation "com.google.guava:guava:18.0" 8 | testImplementation "org.embulk:embulk-formatter-csv:0.11.1" 9 | testImplementation "org.embulk:embulk-input-file:0.11.0" 10 | testImplementation "org.embulk:embulk-output-file:0.11.0" 11 | testImplementation "org.embulk:embulk-parser-csv:0.11.3" 12 | 13 | testImplementation "mysql:mysql-connector-java:5.1.44" 14 | } 15 | 16 | embulkPlugin { 17 | mainClass = "org.embulk.input.mysql.MySQLInputPlugin" 18 | category = "input" 19 | type = "mysql" 20 | additionalDependencyDeclarations = [ 21 | [ groupId: "mysql", artifactId: "mysql-connector-java", version: "5.1.44", scope: "compile", optional: true ], 22 | ] 23 | } 24 | 25 | publishing { 26 | publications { 27 | maven(MavenPublication) { 28 | pom { // https://central.sonatype.org/pages/requirements.html 29 | developers { 30 | developer { 31 | name = "Sadayuki Furuhashi" 32 | email = "frsyuki@gmail.com" 33 | } 34 | developer { 35 | name = "Hitoshi Tanaka" 36 | email = "thitoshi@cac.co.jp" 37 | } 38 | developer { 39 | name = "Muga Nishizawa" 40 | email = "muga.nishizawa@gmail.com" 41 | } 42 | developer { 43 | name = "Hiroyuki Sato" 44 | email = "hiroysato@gmail.com" 45 | } 46 | developer { 47 | name = "Dai MIKURUBE" 48 | email = "dmikurube@treasure-data.com" 49 | } 50 | developer { 51 | name = "Robert Nguyen" 52 | email = "ng.hung83@gmail.com" 53 | } 54 | } 55 | } 56 | } 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /embulk-input-mysql/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 9 | mysql:mysql-connector-java:5.1.44=compileClasspath 10 | org.embulk:embulk-spi:0.11=compileClasspath 11 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 12 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 13 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 14 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 15 | org.msgpack:msgpack-core:0.8.24=compileClasspath 16 | org.slf4j:slf4j-api:2.0.7=compileClasspath 17 | empty= 18 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/main/java/org/embulk/input/mysql/getter/AbstractMySQLTimestampIncrementalHandler.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.mysql.getter; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import org.embulk.config.ConfigSource; 5 | import org.embulk.input.jdbc.getter.AbstractIncrementalHandler; 6 | import org.embulk.input.jdbc.getter.ColumnGetter; 7 | import org.embulk.spi.Column; 8 | import org.embulk.spi.Exec; 9 | import org.embulk.util.config.Task; 10 | import org.embulk.util.timestamp.TimestampFormatter; 11 | 12 | import java.sql.PreparedStatement; 13 | import java.sql.ResultSet; 14 | import java.sql.SQLException; 15 | import java.sql.Timestamp; 16 | import java.time.Instant; 17 | import java.time.ZoneId; 18 | import java.util.Optional; 19 | 20 | public abstract class AbstractMySQLTimestampIncrementalHandler 21 | extends AbstractIncrementalHandler 22 | { 23 | protected final ZoneId sessionTimeZone; 24 | protected long epochSecond; 25 | protected int nano; 26 | 27 | public AbstractMySQLTimestampIncrementalHandler(final ZoneId sessionTimeZone, final ColumnGetter next) 28 | { 29 | super(next); 30 | this.sessionTimeZone = sessionTimeZone; 31 | } 32 | 33 | @Override 34 | public void getAndSet(ResultSet from, int fromIndex, Column toColumn) 35 | throws SQLException 36 | { 37 | Timestamp timestamp = from.getTimestamp(fromIndex); 38 | if (timestamp != null) { 39 | epochSecond = timestamp.getTime() / 1000; 40 | nano = timestamp.getNanos(); 41 | } 42 | 43 | super.getAndSet(from, fromIndex, toColumn); 44 | } 45 | 46 | @Override 47 | public JsonNode encodeToJson() 48 | { 49 | final TimestampFormatter formatter = TimestampFormatter.builder(getTimestampFormat(), true).build(); 50 | String text = formatter.format(utcTimestampFromSessionTime(epochSecond, nano)); 51 | return jsonNodeFactory.textNode(text); 52 | } 53 | 54 | protected abstract String getTimestampFormat(); 55 | 56 | protected abstract Instant utcTimestampFromSessionTime(long epochSecond, int nano); 57 | 58 | @Override 59 | public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue) 60 | throws SQLException 61 | { 62 | final TimestampFormatter formatter = TimestampFormatter.builder(getTimestampPattern(), true).build(); 63 | final Instant epoch = formatter.parse(fromValue.asText()); 64 | toStatement.setTimestamp(toIndex, utcTimestampToSessionTime(epoch)); 65 | } 66 | 67 | protected abstract String getTimestampPattern(); 68 | 69 | protected abstract Timestamp utcTimestampToSessionTime(Instant ts); 70 | } 71 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/main/java/org/embulk/input/mysql/getter/MySQLColumnGetterFactory.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.mysql.getter; 2 | 3 | import org.embulk.config.ConfigException; 4 | import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask; 5 | import org.embulk.input.jdbc.JdbcColumn; 6 | import org.embulk.input.jdbc.JdbcColumnOption; 7 | import org.embulk.input.jdbc.JdbcInputConnection; 8 | import org.embulk.input.jdbc.getter.ColumnGetter; 9 | import org.embulk.input.jdbc.getter.ColumnGetterFactory; 10 | import org.embulk.input.jdbc.getter.JsonColumnGetter; 11 | import org.embulk.input.mysql.MySQLInputConnection; 12 | import org.embulk.spi.PageBuilder; 13 | import org.embulk.spi.type.Types; 14 | 15 | import java.time.ZoneId; 16 | import java.util.TimeZone; 17 | 18 | public class MySQLColumnGetterFactory 19 | extends ColumnGetterFactory 20 | { 21 | public MySQLColumnGetterFactory(final PageBuilder to, final ZoneId defaultTimeZone) 22 | { 23 | super(to, defaultTimeZone); 24 | } 25 | 26 | @Override 27 | public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option) 28 | { 29 | ColumnGetter getter = super.newColumnGetter(con, task, column, option); 30 | 31 | switch (column.getTypeName()) { 32 | case "DATETIME": 33 | case "TIMESTAMP": 34 | int index = task.getQuerySchema().findColumn(column.getName()).get(); 35 | if (!task.getIncremental() || !task.getIncrementalColumnIndexes().contains(index)) { 36 | return getter; 37 | } 38 | 39 | // incremental loading 40 | MySQLInputConnection mysqlInputConnection = (MySQLInputConnection) con; 41 | // Users cannot use DATETIME or TIMESTAMP typed columns as incremental_columns: if 'useLegacyDatetimeCode=true'. 42 | // That might be acceptable since mysql-connector-java v6.x will turn off, by default. 43 | if (mysqlInputConnection.getUseLegacyDatetimeCode()) { 44 | throw new ConfigException("Must use 'useLegacyDatetimeCode=false' if 'DATETIME' or 'TIMESTAMP' typed columns are used as incremental_columns:"); 45 | } 46 | 47 | TimeZone timeZone = mysqlInputConnection.getServerTimezoneTZ(); 48 | final ZoneId sessionTimeZone = timeZone.toZoneId(); 49 | if (column.getTypeName().equals("DATETIME")) { 50 | return new MySQLDateTimeTimestampIncrementalHandler(sessionTimeZone, getter); 51 | } 52 | else { // TIMESTAMP 53 | return new MySQLTimestampTimestampIncrementalHandler(sessionTimeZone, getter); 54 | } 55 | case "JSON": 56 | return new JsonColumnGetter(to, Types.JSON); 57 | default: 58 | return getter; 59 | } 60 | } 61 | 62 | @Override 63 | protected String sqlTypeToValueType(JdbcColumn column, int sqlType) 64 | { 65 | if ("json".equals(column.getTypeName())) { 66 | return "json"; 67 | } 68 | return super.sqlTypeToValueType(column, sqlType); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/main/java/org/embulk/input/mysql/getter/MySQLDateTimeTimestampIncrementalHandler.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.mysql.getter; 2 | 3 | import org.embulk.input.jdbc.getter.ColumnGetter; 4 | 5 | import java.sql.Timestamp; 6 | import java.time.Instant; 7 | import java.time.ZoneId; 8 | import java.time.ZoneOffset; 9 | import java.time.ZonedDateTime; 10 | 11 | public class MySQLDateTimeTimestampIncrementalHandler 12 | extends AbstractMySQLTimestampIncrementalHandler 13 | { 14 | public MySQLDateTimeTimestampIncrementalHandler(final ZoneId sessionTimeZone, final ColumnGetter next) 15 | { 16 | super(sessionTimeZone, next); 17 | } 18 | 19 | @Override 20 | public String getTimestampFormat() 21 | { 22 | return "%Y-%m-%dT%H:%M:%S.%6N"; 23 | } 24 | 25 | @Override 26 | public Instant utcTimestampFromSessionTime(final long epochSecond, final int nano) 27 | { 28 | // this Timestamp value is already converted by session time_zone. 29 | final long reconverted = ZonedDateTime.ofInstant(Instant.ofEpochSecond(epochSecond), this.sessionTimeZone). 30 | withZoneSameLocal(ZoneOffset.UTC).toEpochSecond(); // reconvert from session time_zone to UTC 31 | return Instant.ofEpochSecond(reconverted, nano); 32 | } 33 | 34 | @Override 35 | public String getTimestampPattern() 36 | { 37 | return "%Y-%m-%dT%H:%M:%S.%N"; 38 | } 39 | 40 | @Override 41 | public Timestamp utcTimestampToSessionTime(final Instant from) 42 | { 43 | // reconvert from UTC to session time_zone 44 | final long reconverted = ZonedDateTime.ofInstant(from, ZoneOffset.UTC) 45 | .withZoneSameLocal(this.sessionTimeZone).toEpochSecond() * 1000; 46 | Timestamp sqlTimestamp = new Timestamp(reconverted); 47 | sqlTimestamp.setNanos(from.getNano()); 48 | return sqlTimestamp; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/main/java/org/embulk/input/mysql/getter/MySQLTimestampTimestampIncrementalHandler.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.mysql.getter; 2 | 3 | import org.embulk.input.jdbc.getter.ColumnGetter; 4 | 5 | import java.sql.Timestamp; 6 | import java.time.Instant; 7 | import java.time.ZoneId; 8 | 9 | public class MySQLTimestampTimestampIncrementalHandler 10 | extends AbstractMySQLTimestampIncrementalHandler 11 | { 12 | public MySQLTimestampTimestampIncrementalHandler(final ZoneId sessionTimeZone, final ColumnGetter next) 13 | { 14 | super(sessionTimeZone, next); 15 | } 16 | 17 | @Override 18 | public String getTimestampFormat() 19 | { 20 | return "%Y-%m-%dT%H:%M:%S.%6NZ"; 21 | } 22 | 23 | @Override 24 | public Instant utcTimestampFromSessionTime(final long epochSecond, final int nano) 25 | { 26 | return Instant.ofEpochSecond(epochSecond, nano); 27 | } 28 | 29 | @Override 30 | public String getTimestampPattern() 31 | { 32 | return "%Y-%m-%dT%H:%M:%S.%N%z"; 33 | } 34 | 35 | @Override 36 | public Timestamp utcTimestampToSessionTime(final Instant ts) 37 | { 38 | Timestamp sqlTimestamp = new Timestamp(ts.getEpochSecond() * 1000); 39 | sqlTimestamp.setNanos(ts.getNano()); 40 | return sqlTimestamp; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/java/org/embulk/input/mysql/MySQLColumnGetterFactoryTest.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.mysql; 2 | 3 | import com.fasterxml.jackson.databind.node.ObjectNode; 4 | import java.util.Optional; 5 | import java.time.ZoneId; 6 | import org.embulk.config.TaskSource; 7 | import org.embulk.input.jdbc.JdbcColumn; 8 | import org.embulk.input.jdbc.JdbcColumnOption; 9 | import org.embulk.input.jdbc.getter.ColumnGetterFactory; 10 | import org.embulk.input.jdbc.getter.JsonColumnGetter; 11 | import org.embulk.input.jdbc.getter.StringColumnGetter; 12 | import org.embulk.input.mysql.getter.MySQLColumnGetterFactory; 13 | import org.embulk.spi.type.Type; 14 | import org.junit.Test; 15 | 16 | import static org.hamcrest.MatcherAssert.assertThat; 17 | import static org.hamcrest.core.IsInstanceOf.instanceOf; 18 | 19 | public class MySQLColumnGetterFactoryTest 20 | { 21 | @Test 22 | public void testJsonSupported() 23 | { 24 | JdbcColumnOption mockColumnOption = new JdbcColumnOption() 25 | { 26 | @Override 27 | public String getValueType() 28 | { 29 | return "coalesce"; 30 | } 31 | 32 | @Override 33 | public Optional getType() 34 | { 35 | return Optional.empty(); 36 | } 37 | 38 | @Override 39 | public Optional getTimestampFormat() 40 | { 41 | return Optional.empty(); 42 | } 43 | 44 | @Override 45 | public Optional getTimeZone() 46 | { 47 | return Optional.empty(); 48 | } 49 | 50 | @Override 51 | public void validate() 52 | { 53 | 54 | } 55 | 56 | @Override 57 | @SuppressWarnings("deprecation") 58 | public TaskSource dump() 59 | { 60 | return null; 61 | } 62 | 63 | @Override 64 | public TaskSource toTaskSource() 65 | { 66 | return null; 67 | } 68 | 69 | @Override 70 | public ObjectNode toObjectNode() 71 | { 72 | return null; 73 | } 74 | }; 75 | JdbcColumn jsonColumn = new JdbcColumn("any_column_name", "JSON", 1, 0, 0); // 1: CHAR 76 | 77 | ColumnGetterFactory fac = new MySQLColumnGetterFactory(null, null); 78 | assertThat(fac.newColumnGetter(null, null, jsonColumn, mockColumnOption), instanceOf(JsonColumnGetter.class)); 79 | 80 | fac = new ColumnGetterFactory(null, null); 81 | assertThat(fac.newColumnGetter(null, null, jsonColumn, mockColumnOption), instanceOf(StringColumnGetter.class)); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/java/org/embulk/input/mysql/MySQLTests.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.mysql; 2 | 3 | import com.google.common.base.Throwables; 4 | import com.google.common.collect.ImmutableList; 5 | import com.google.common.io.ByteStreams; 6 | 7 | import org.embulk.config.ConfigSource; 8 | import org.embulk.test.EmbulkTests; 9 | 10 | import java.io.IOException; 11 | 12 | import static java.util.Locale.ENGLISH; 13 | 14 | public class MySQLTests 15 | { 16 | public static ConfigSource baseConfig() 17 | { 18 | return EmbulkTests.config("EMBULK_INPUT_MYSQL_TEST_CONFIG"); 19 | } 20 | 21 | public static void execute(String sql) 22 | { 23 | ConfigSource config = baseConfig(); 24 | 25 | ImmutableList.Builder args = ImmutableList.builder(); 26 | args.add("mysql") 27 | .add("-u") 28 | .add(config.get(String.class, "user")); 29 | if (!config.get(String.class, "password").isEmpty()) { 30 | args.add("-p" + config.get(String.class, "password")); 31 | } 32 | args 33 | .add("-h") 34 | .add(config.get(String.class, "host")) 35 | .add("-P") 36 | .add(config.get(String.class, "port", "3306")) 37 | .add(config.get(String.class, "database")) 38 | .add("-e") 39 | .add(convert(sql)); 40 | 41 | ProcessBuilder pb = new ProcessBuilder(args.build()); 42 | pb.redirectErrorStream(true); 43 | int code; 44 | try { 45 | Process process = pb.start(); 46 | ByteStreams.copy(process.getInputStream(), System.out); 47 | code = process.waitFor(); 48 | } catch (IOException | InterruptedException ex) { 49 | throw Throwables.propagate(ex); 50 | } 51 | if (code != 0) { 52 | throw new RuntimeException(String.format(ENGLISH, 53 | "Command finished with non-zero exit code. Exit code is %d.", code)); 54 | } 55 | } 56 | 57 | private static String convert(String sql) 58 | { 59 | if (System.getProperty("os.name").contains("Windows")) { 60 | // '"' should be '\"' and '\' should be '\\' in Windows 61 | return sql.replace("\\\\", "\\").replace("\\", "\\\\").replace("\"", "\\\""); 62 | } 63 | return sql; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | 3 | create table test1 ( 4 | id char(2), 5 | c1 tinyint, 6 | c2 smallint, 7 | c3 int, 8 | c4 bigint, 9 | c5 float, 10 | c6 double, 11 | c7 decimal(4,0), 12 | c8 decimal(20,2), 13 | c9 char(4), 14 | c10 varchar(4), 15 | c11 date, 16 | c12 datetime, 17 | c13 timestamp, 18 | c14 time, 19 | c15 datetime(6), 20 | c16 json, 21 | primary key(id) 22 | ); 23 | 24 | insert into test1 values( 25 | '10', 26 | null, 27 | null, 28 | null, 29 | null, 30 | null, 31 | null, 32 | null, 33 | null, 34 | null, 35 | null, 36 | null, 37 | null, 38 | '2015-06-04 23:45:06', 39 | null, 40 | null, 41 | null 42 | ); 43 | 44 | insert into test1 values( 45 | '11', 46 | 99, 47 | 9999, 48 | -99999999, 49 | -9999999999999999, 50 | 1.2345, 51 | 1.234567890123, 52 | -1234, 53 | 123456789012345678.12, 54 | '5678', 55 | 'xy', 56 | '2015-06-04', 57 | '2015-06-04 12:34:56', 58 | '2015-06-04 23:45:06', 59 | '08:04:02', 60 | '2015-06-04 01:02:03.123456', 61 | '{"key":"value"}' 62 | ); 63 | 64 | drop table if exists test2; 65 | 66 | create table test2 (c1 bigint unsigned); 67 | 68 | insert into test2 values(18446744073709551615); -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_boolean_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | column_options: 4 | c1: {type: boolean} 5 | c2: {type: boolean} 6 | c3: {type: boolean} 7 | c4: {type: boolean} 8 | c5: {type: boolean} 9 | c6: {type: boolean} 10 | c7: {type: boolean} 11 | c8: {type: boolean} 12 | c9: {type: boolean} 13 | c10: {type: boolean} 14 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_boolean_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,true,true,false,false,true,true,false,true,,,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_boolean_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_double_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | column_options: 4 | c1: {type: double} 5 | c2: {type: double} 6 | c3: {type: double} 7 | c4: {type: double} 8 | c5: {type: double} 9 | c6: {type: double} 10 | c7: {type: double} 11 | c8: {type: double} 12 | c9: {type: double} 13 | c10: {type: double} 14 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_double_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,99.0,9999.0,-9.9999999E7,-1.0E16,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678.0,,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_double_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_invalid_zone_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | column_options: 4 | c11: {type: string, timestamp_format: '%Y/%m/%d'} 5 | c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'} 6 | c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: 'Somewhere/Some_City'} 7 | c14: {type: string, timestamp_format: '%H-%M-%S', timezone: 'UTC'} 8 | c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'} 9 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_long_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | column_options: 4 | c1: {type: long} 5 | c2: {type: long} 6 | c3: {type: long} 7 | c4: {type: long} 8 | c5: {type: long} 9 | c6: {type: long} 10 | c7: {type: long} 11 | c8: {type: long} 12 | c9: {type: long} 13 | c10: {type: long} 14 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_long_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,99,9999,-99999999,-9999999999999999,1,1,-1234,123456789012345678,5678,,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_long_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_string_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | column_options: 4 | c1: {type: string} 5 | c2: {type: string} 6 | c3: {type: string} 7 | c4: {type: string} 8 | c5: {type: string} 9 | c6: {type: string} 10 | c7: {type: string} 11 | c8: {type: string} 12 | c9: {type: string} 13 | c10: {type: string} 14 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_string_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,99,9999,-99999999,-9999999999999999,1.2345,1.234567890123,-1234,123456789012345678.12,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_string_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp1_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | column_options: 4 | #c11: {type: timestamp, timestamp_format: '%Y/%m/%d'} 5 | #c12: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S'} 6 | #c13: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: '+0900'} 7 | #c14: {type: timestamp, timestamp_format: '%H-%M-%S', timezone: '+0900'} 8 | #c15: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'} 9 | c11: {type: timestamp} 10 | c12: {type: timestamp} 11 | c13: {type: timestamp} 12 | c14: {type: timestamp} 13 | c15: {type: timestamp} 14 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp1_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp1_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp2_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | column_options: 4 | c11: {type: string, timestamp_format: '%Y/%m/%d'} 5 | c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'} 6 | c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: '+0900'} 7 | c14: {type: string, timestamp_format: '%H-%M-%S', timezone: '+0900'} 8 | c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'} 9 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp2_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015/06/05 08:45:06,,, 2 | 11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/03,2015/06/04 12:34:56,2015/06/05 08:45:06,15-04-02,2015/06/04 01:02:03.123456,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp2_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp3_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | default_timezone: '+0900' 4 | column_options: 5 | c11: {type: string, timestamp_format: '%Y/%m/%d'} 6 | c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'} 7 | c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'} 8 | c14: {type: string, timestamp_format: '%H-%M-%S', timezone: '+0300'} 9 | c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'} 10 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp3_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015/06/05 08:45:06,,, 2 | 11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/04,2015/06/04 21:34:56,2015/06/05 08:45:06,09-04-02,2015/06/04 10:02:03.123456,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp3_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_decimal_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | column_options: 3 | c1: {value_type: decimal} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_decimal_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,99.0,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_decimal_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_json_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | column_options: 3 | c10: {value_type: json} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_json_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_json_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_string_config.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | column_options: 3 | c1: {value_type: string} 4 | c16: {value_type: string} 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_string_expected.csv: -------------------------------------------------------------------------------- 1 | 10,,,,,,,,,,,,,2015-06-04 23:45:06.000000 +0000,,, 2 | 11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 12:34:56.000000 +0000,2015-06-04 23:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-04 01:02:03.123456 +0000,"{""key"":""value""}" 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_string_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/char/config_1.yml: -------------------------------------------------------------------------------- 1 | table: char_load 2 | incremental: true 3 | incremental_columns: [name] 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/char/config_2.yml: -------------------------------------------------------------------------------- 1 | table: char_load 2 | last_record: ['A4'] 3 | incremental: true 4 | incremental_columns: [name] 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/char/expected_1.csv: -------------------------------------------------------------------------------- 1 | A1,first 2 | A2,first 3 | A3,first 4 | A4,first 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/char/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['A4'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/char/expected_2.csv: -------------------------------------------------------------------------------- 1 | A5,more_load 2 | A9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/char/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['A9'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/char/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into char_load (name, note) values 3 | ('A0', 'more_skip'), 4 | ('A4', 'more_skip'), 5 | ('A9', 'more_load'), 6 | ('A5', 'more_load'); 7 | 8 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/char/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists char_load; 2 | 3 | create table char_load ( 4 | name char(2) not null, 5 | note text 6 | ); 7 | 8 | insert into char_load (name, note) values 9 | ('A3', 'first'), 10 | ('A4', 'first'), 11 | ('A2', 'first'), 12 | ('A1', 'first'); 13 | 14 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/config_1.yml: -------------------------------------------------------------------------------- 1 | table: dt_load 2 | default_time_zone: +0300 # should be ignored 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/config_2.yml: -------------------------------------------------------------------------------- 1 | table: dt_load 2 | last_record: ['2016-11-02T04:00:05.333003'] 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/expected_1.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 01:00:01.000000 +0000,first 2 | 2016-11-02 02:00:02.000000 +0000,first 3 | 2016-11-02 03:00:03.000000 +0000,first 4 | 2016-11-02 04:00:04.000000 +0000,first 5 | 2016-11-02 04:00:05.111001 +0000,first 6 | 2016-11-02 04:00:05.222002 +0000,first 7 | 2016-11-02 04:00:05.333003 +0000,first 8 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:05.333003'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/expected_2.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 04:00:05.333004 +0000,more_load 2 | 2016-11-02 04:00:06.000000 +0000,more_load 3 | 2016-11-02 04:00:06.000000 +0000,more_load 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:06.000000'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into dt_load (time, note) values 3 | ('2016-11-02 04:00:00', 'more_skip'), 4 | ('2016-11-02 04:00:05.333000', 'more_skip'), 5 | ('2016-11-02 04:00:05.333003', 'more_skip'), 6 | ('2016-11-02 04:00:05.333004', 'more_load'), 7 | ('2016-11-02 04:00:06', 'more_load'), 8 | ('2016-11-02 04:00:06', 'more_load'); 9 | 10 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists dt_load; 2 | 3 | create table dt_load ( 4 | time datetime(6) not null, 5 | note text 6 | ); 7 | 8 | insert into dt_load (time, note) values 9 | ('2016-11-02 01:00:01', 'first'), 10 | ('2016-11-02 02:00:02', 'first'), 11 | ('2016-11-02 03:00:03', 'first'), 12 | ('2016-11-02 04:00:04', 'first'), 13 | ('2016-11-02 04:00:05.111001', 'first'), 14 | ('2016-11-02 04:00:05.222002', 'first'), 15 | ('2016-11-02 04:00:05.333003', 'first'); 16 | 17 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/int/config_1.yml: -------------------------------------------------------------------------------- 1 | table: int_load 2 | incremental: true 3 | incremental_columns: [num] 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/int/config_2.yml: -------------------------------------------------------------------------------- 1 | table: int_load 2 | last_record: [4] 3 | incremental: true 4 | incremental_columns: [num] 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/int/expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,first 2 | 2,first 3 | 3,first 4 | 4,first 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/int/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/int/expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,more_load 2 | 9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/int/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/int/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into int_load (num, note) values 3 | (0, 'more_skip'), 4 | (4, 'more_skip'), 5 | (9, 'more_load'), 6 | (5, 'more_load'); 7 | 8 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/int/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists int_load; 2 | 3 | create table int_load ( 4 | num int not null, 5 | note text 6 | ); 7 | 8 | insert into int_load (num, note) values 9 | (3, 'first'), 10 | (4, 'first'), 11 | (2, 'first'), 12 | (1, 'first'); 13 | 14 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0] 2 | incremental: true 3 | incremental_columns: [num] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | num,note 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | ORDER BY 14 | num ASC 15 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4] 2 | incremental: true 3 | incremental_columns: [num] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | num,note 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | ORDER BY 14 | num ASC 15 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,first 2 | 2,first 3 | 3,first 4 | 4,first 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,more_load 2 | 9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into query_load (num, num2, note) values 3 | (0, 100, 'more_skip'), 4 | (4, 104, 'more_skip'), 5 | (9, 109, 'more_load'), 6 | (5, 105, 'more_load'); 7 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/multi_columns_config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0,0] 2 | incremental: true 3 | incremental_columns: [num,num2] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | * 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | OR (num = :num AND num2 > :num2) 14 | ORDER BY 15 | num ASC, 16 | num2 ASC 17 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/multi_columns_config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4,104] 2 | incremental: true 3 | incremental_columns: [num,num2] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | * 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | AND num2 IS NOT NULL 14 | OR (num = :num AND num2 > :num2) 15 | ORDER BY 16 | num ASC, 17 | num2 ASC 18 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/multi_columns_expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,101,first 2 | 2,102,first 3 | 3,103,first 4 | 4,104,first 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/multi_columns_expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4,104] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/multi_columns_expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,105,more_load 2 | 9,109,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/multi_columns_expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9,109] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/same_length_placeholders_config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0,0] 2 | incremental: true 3 | incremental_columns: [num,val] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | num, num2 AS val, note 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | OR (num = :num AND num2 > :val) 14 | ORDER BY 15 | num ASC, 16 | val ASC 17 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/same_length_placeholders_config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4,104] 2 | incremental: true 3 | incremental_columns: [num,val] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | num, num2 AS val, note 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | AND num2 IS NOT NULL 14 | OR (num = :num AND num2 > :val) 15 | ORDER BY 16 | num ASC, 17 | val ASC 18 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/query/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists query_load; 2 | 3 | create table query_load ( 4 | num int not null, 5 | num2 int not null, 6 | note text 7 | ); 8 | 9 | insert into query_load (num, num2, note) values 10 | (3, 103, 'first'), 11 | (4, 104, 'first'), 12 | (2, 102, 'first'), 13 | (1, 101, 'first'); 14 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/ts/config_1.yml: -------------------------------------------------------------------------------- 1 | table: ts_load 2 | default_time_zone: +0300 # should be ignored 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/ts/config_2.yml: -------------------------------------------------------------------------------- 1 | table: ts_load 2 | last_record: ['2016-11-02T04:00:05.333003Z'] 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/ts/expected_1.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 01:00:01.000000 +0000,first 2 | 2016-11-02 02:00:02.000000 +0000,first 3 | 2016-11-02 03:00:03.000000 +0000,first 4 | 2016-11-02 04:00:04.000000 +0000,first 5 | 2016-11-02 04:00:05.111001 +0000,first 6 | 2016-11-02 04:00:05.222002 +0000,first 7 | 2016-11-02 04:00:05.333003 +0000,first 8 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/ts/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:05.333003Z'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/ts/expected_2.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 04:00:05.333004 +0000,more_load 2 | 2016-11-02 04:00:06.000000 +0000,more_load 3 | 2016-11-02 04:00:06.000000 +0000,more_load 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/ts/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:06.000000Z'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/ts/insert_more.sql: -------------------------------------------------------------------------------- 1 | set time_zone = '+00:00'; 2 | 3 | insert into ts_load (time, note) values 4 | ('2016-11-02 04:00:00', 'more_skip'), 5 | ('2016-11-02 04:00:05.333000', 'more_skip'), 6 | ('2016-11-02 04:00:05.333003', 'more_skip'), 7 | ('2016-11-02 04:00:05.333004', 'more_load'), 8 | ('2016-11-02 04:00:06', 'more_load'), 9 | ('2016-11-02 04:00:06', 'more_load'); 10 | 11 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/incremental/ts/setup.sql: -------------------------------------------------------------------------------- 1 | set time_zone = '+00:00'; 2 | 3 | drop table if exists ts_load; 4 | 5 | create table ts_load ( 6 | time timestamp(6) not null, 7 | note text 8 | ); 9 | 10 | insert into ts_load (time, note) values 11 | ('2016-11-02 01:00:01', 'first'), 12 | ('2016-11-02 02:00:02', 'first'), 13 | ('2016-11-02 03:00:03', 'first'), 14 | ('2016-11-02 04:00:04', 'first'), 15 | ('2016-11-02 04:00:05.111001', 'first'), 16 | ('2016-11-02 04:00:05.222002', 'first'), 17 | ('2016-11-02 04:00:05.333003', 'first'); 18 | 19 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/after_select.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | after_select: delete from test1 where id = 3 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/after_select_expected.csv: -------------------------------------------------------------------------------- 1 | 1,A 2 | 2,B 3 | 4,D 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/before_select.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | before_select: delete from test1 where id = 3 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/before_select_expected.csv: -------------------------------------------------------------------------------- 1 | 1,A 2 | 2,B 3 | 4,D 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/before_setup.yml: -------------------------------------------------------------------------------- 1 | table: test2 2 | order_by: 'id' 3 | before_setup: create table test2 (id int, num int); insert into test2 values(1, 999), (2, 0); 4 | d: insert into test2 values(1, 999), (2, 0); 5 | options: 6 | allowMultiQueries: true -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/before_setup_expected.csv: -------------------------------------------------------------------------------- 1 | 1,999 2 | 2,0 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/column_options.yml: -------------------------------------------------------------------------------- 1 | table: test3 2 | order_by: 'id' 3 | column_options: 4 | datetime1: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: "+0000"} 5 | datetime2: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 6 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/column_options_expected.csv: -------------------------------------------------------------------------------- 1 | 1,, 2 | 2,2019/01/02 12:34:56,2019-01-02 21:34:56 3 | 3,2018/12/31 23:59:59,2019-01-01 08:59:59 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/default_column_options.yml: -------------------------------------------------------------------------------- 1 | table: test3 2 | order_by: 'id' 3 | default_column_options: 4 | TIMESTAMP: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: "+0000"} 5 | column_options: 6 | datetime2: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 7 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/default_column_options_expected.csv: -------------------------------------------------------------------------------- 1 | 1,, 2 | 2,2019/01/02 12:34:56,2019-01-02 21:34:56 3 | 3,2018/12/31 23:59:59,2019-01-01 08:59:59 4 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/expected1.csv: -------------------------------------------------------------------------------- 1 | 1,A 2 | 2,B 3 | 3,C 4 | 4,D 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/input.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/order_by.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id' 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/order_by_asc.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id asc' 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/order_by_asc_expected.csv: -------------------------------------------------------------------------------- 1 | 1,A 2 | 2,B 3 | 3,C 4 | 4,D 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/order_by_desc.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | order_by: 'id desc' 3 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/order_by_desc_expected.csv: -------------------------------------------------------------------------------- 1 | 4,D 2 | 3,C 3 | 2,B 4 | 1,A 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/order_by_expected.csv: -------------------------------------------------------------------------------- 1 | 1,A 2 | 2,B 3 | 3,C 4 | 4,D 5 | -------------------------------------------------------------------------------- /embulk-input-mysql/src/test/resources/org/embulk/input/mysql/test/expect/option/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | 3 | create table test1 ( 4 | id int not null, 5 | name varchar(8) not null, 6 | primary key(id) 7 | ); 8 | 9 | insert into test1 values 10 | (1, 'A'), 11 | (2, 'B'), 12 | (3, 'C'), 13 | (4, 'D'); 14 | 15 | drop table if exists test2; 16 | 17 | drop table if exists test3; 18 | 19 | create table test3 ( 20 | id int not null, 21 | datetime1 datetime, 22 | datetime2 datetime, 23 | primary key(id) 24 | ); 25 | 26 | insert into test3 values 27 | (1, NULL, NULL), 28 | (2, '2019-01-02 12:34:56', '2019-01-02 12:34:56'), 29 | (3, '2018-12-31 23:59:59', '2018-12-31 23:59:59'); 30 | -------------------------------------------------------------------------------- /embulk-input-postgresql/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation(project(path: ":embulk-input-jdbc", configuration: "runtimeElements")) 3 | 4 | compileOnly "org.postgresql:postgresql:9.4-1205-jdbc41" 5 | defaultJdbcDriver 'org.postgresql:postgresql:9.4-1205-jdbc41' 6 | 7 | testImplementation "com.google.guava:guava:18.0" 8 | testImplementation "org.embulk:embulk-formatter-csv:0.11.1" 9 | testImplementation "org.embulk:embulk-input-file:0.11.0" 10 | testImplementation "org.embulk:embulk-output-file:0.11.0" 11 | testImplementation "org.embulk:embulk-parser-csv:0.11.3" 12 | 13 | testImplementation "org.postgresql:postgresql:9.4-1205-jdbc41" 14 | } 15 | 16 | embulkPlugin { 17 | mainClass = "org.embulk.input.postgresql.PostgreSQLInputPlugin" 18 | category = "input" 19 | type = "postgresql" 20 | additionalDependencyDeclarations = [ 21 | [ groupId: "org.postgresql", artifactId: "postgresql", version: "9.4-1205-jdbc41", scope: "compile", optional: true ], 22 | ] 23 | } 24 | 25 | publishing { 26 | publications { 27 | maven(MavenPublication) { 28 | pom { // https://central.sonatype.org/pages/requirements.html 29 | developers { 30 | developer { 31 | name = "Sadayuki Furuhashi" 32 | email = "frsyuki@gmail.com" 33 | } 34 | developer { 35 | name = "Hitoshi Tanaka" 36 | email = "thitoshi@cac.co.jp" 37 | } 38 | developer { 39 | name = "Tomohiro Hashidate" 40 | email = "kakyoin.hierophant@gmail.com" 41 | } 42 | developer { 43 | name = "You Yamagata" 44 | email = "youy.bg8@gmail.com" 45 | } 46 | developer { 47 | name = "Shinichi Ishimura" 48 | email = "shiketaudonko41@gmail.com" 49 | } 50 | developer { 51 | name = "Muga Nishizawa" 52 | email = "muga.nishizawa@gmail.com" 53 | } 54 | developer { 55 | name = "Khoa Nguyen" 56 | email = "instcode@gmail.com" 57 | } 58 | developer { 59 | name = "Trung Huynh" 60 | email = "httrung90@gmail.com" 61 | } 62 | developer { 63 | name = "Satoshi Akama" 64 | email = "satoshiakama@gmail.com" 65 | } 66 | developer { 67 | name = "Dai MIKURUBE" 68 | email = "dmikurube@treasure-data.com" 69 | } 70 | } 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /embulk-input-postgresql/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 9 | org.embulk:embulk-spi:0.11=compileClasspath 10 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 11 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 12 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 13 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 14 | org.msgpack:msgpack-core:0.8.24=compileClasspath 15 | org.postgresql:postgresql:9.4-1205-jdbc41=compileClasspath 16 | org.slf4j:slf4j-api:2.0.7=compileClasspath 17 | empty= 18 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/main/java/org/embulk/input/postgresql/getter/HstoreToJsonColumnGetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.postgresql.getter; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.embulk.input.jdbc.getter.JsonColumnGetter; 6 | import org.embulk.spi.Column; 7 | import org.embulk.spi.PageBuilder; 8 | import org.embulk.spi.type.Type; 9 | import org.embulk.util.json.JsonParseException; 10 | import org.msgpack.value.Value; 11 | import org.postgresql.util.HStoreConverter; 12 | 13 | import java.util.Map; 14 | 15 | public class HstoreToJsonColumnGetter 16 | extends JsonColumnGetter 17 | { 18 | private final ObjectMapper mapper = new ObjectMapper(); 19 | 20 | public HstoreToJsonColumnGetter(PageBuilder to, Type toType) 21 | { 22 | super(to, toType); 23 | } 24 | 25 | @Override 26 | public void jsonColumn(Column column) 27 | { 28 | Value v; 29 | try { 30 | Map map = HStoreConverter.fromString(value); 31 | v = jsonParser.parse(mapper.writeValueAsString(map)); 32 | } catch (JsonProcessingException | JsonParseException e) { 33 | super.jsonColumn(column); 34 | return; 35 | } 36 | to.setJson(column, v); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/main/java/org/embulk/input/postgresql/getter/PostgreSQLColumnGetterFactory.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.postgresql.getter; 2 | 3 | import java.time.ZoneId; 4 | import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask; 5 | import org.embulk.input.jdbc.JdbcColumn; 6 | import org.embulk.input.jdbc.JdbcColumnOption; 7 | import org.embulk.input.jdbc.JdbcInputConnection; 8 | import org.embulk.input.jdbc.getter.ColumnGetter; 9 | import org.embulk.input.jdbc.getter.ColumnGetterFactory; 10 | import org.embulk.input.jdbc.getter.StringColumnGetter; 11 | import org.embulk.input.jdbc.getter.TimestampWithTimeZoneIncrementalHandler; 12 | import org.embulk.input.jdbc.getter.TimestampWithoutTimeZoneIncrementalHandler; 13 | import org.embulk.spi.PageBuilder; 14 | import org.embulk.spi.type.Types; 15 | 16 | public class PostgreSQLColumnGetterFactory extends ColumnGetterFactory 17 | { 18 | public PostgreSQLColumnGetterFactory(final PageBuilder to, final ZoneId defaultTimeZone) 19 | { 20 | super(to, defaultTimeZone); 21 | } 22 | 23 | @Override 24 | public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option) 25 | { 26 | if (column.getTypeName().equals("hstore") && getToType(option) == Types.JSON) { 27 | // converting hstore to json needs a special handling 28 | return new HstoreToJsonColumnGetter(to, Types.JSON); 29 | } 30 | 31 | if (column.getSqlType() == java.sql.Types.ARRAY) { 32 | return new ArrayColumnGetter(to, getToType(option)); 33 | } 34 | 35 | if ("uuid".equals(column.getTypeName())) { 36 | return new StringColumnGetter(to, getToType(option)); 37 | } 38 | 39 | ColumnGetter getter = super.newColumnGetter(con, task, column, option); 40 | 41 | // incremental loading wrapper 42 | switch (column.getTypeName()) { 43 | case "timestamptz": 44 | return new TimestampWithTimeZoneIncrementalHandler(getter); 45 | case "timestamp": 46 | return new TimestampWithoutTimeZoneIncrementalHandler(getter); 47 | default: 48 | return getter; 49 | } 50 | } 51 | 52 | @Override 53 | protected String sqlTypeToValueType(JdbcColumn column, int sqlType) 54 | { 55 | switch(column.getTypeName()) { 56 | case "json": 57 | case "jsonb": 58 | return "json"; 59 | case "hstore": 60 | case "array": 61 | // array & hstore is converted to string by default 62 | return "string"; 63 | default: 64 | return super.sqlTypeToValueType(column, sqlType); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/java/org/embulk/input/postgresql/ArrayTest.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.postgresql; 2 | 3 | import org.embulk.config.ConfigSource; 4 | import org.embulk.formatter.csv.CsvFormatterPlugin; 5 | import org.embulk.input.file.LocalFileInputPlugin; 6 | import org.embulk.output.file.LocalFileOutputPlugin; 7 | import org.embulk.parser.csv.CsvParserPlugin; 8 | import org.embulk.spi.FileInputPlugin; 9 | import org.embulk.spi.FileOutputPlugin; 10 | import org.embulk.spi.FormatterPlugin; 11 | import org.embulk.spi.InputPlugin; 12 | import org.embulk.spi.ParserPlugin; 13 | import org.embulk.test.EmbulkTests; 14 | import org.embulk.test.TestingEmbulk; 15 | import org.junit.Before; 16 | import org.junit.Rule; 17 | import org.junit.Test; 18 | 19 | import java.nio.file.Path; 20 | 21 | import static org.embulk.input.postgresql.PostgreSQLTests.execute; 22 | import static org.embulk.test.EmbulkTests.readSortedFile; 23 | import static org.hamcrest.Matchers.is; 24 | import static org.junit.Assert.assertThat; 25 | 26 | public class ArrayTest 27 | { 28 | private static final String BASIC_RESOURCE_PATH = "/org/embulk/input/postgresql/test/expect/array/"; 29 | 30 | private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) 31 | { 32 | return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName); 33 | } 34 | 35 | private static String readResource(String fileName) 36 | { 37 | return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName); 38 | } 39 | 40 | @Rule 41 | public TestingEmbulk embulk = TestingEmbulk.builder() 42 | .registerPlugin(FileInputPlugin.class, "file", LocalFileInputPlugin.class) 43 | .registerPlugin(ParserPlugin.class, "csv", CsvParserPlugin.class) 44 | .registerPlugin(FormatterPlugin.class, "csv", CsvFormatterPlugin.class) 45 | .registerPlugin(FileOutputPlugin.class, "file", LocalFileOutputPlugin.class) 46 | .registerPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class) 47 | .build(); 48 | 49 | private ConfigSource baseConfig; 50 | 51 | @Before 52 | public void setup() 53 | { 54 | baseConfig = PostgreSQLTests.baseConfig(); 55 | } 56 | 57 | @Test 58 | public void loadAsStringByDefault() throws Exception 59 | { 60 | execute(readResource("setup.sql")); 61 | 62 | Path out1 = embulk.createTempFile("csv"); 63 | embulk.runInput( 64 | baseConfig.merge(loadYamlResource(embulk, "as_string.yml")), 65 | out1); 66 | assertThat( 67 | readSortedFile(out1), 68 | is(readResource("expected_string.csv"))); 69 | } 70 | 71 | @Test 72 | public void loadAsJson() throws Exception 73 | { 74 | execute(readResource("setup.sql")); 75 | 76 | Path out1 = embulk.createTempFile("csv"); 77 | embulk.runInput( 78 | baseConfig.merge(loadYamlResource(embulk, "as_json.yml")), 79 | out1); 80 | assertThat( 81 | readSortedFile(out1), 82 | is(readResource("expected_json.csv"))); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/java/org/embulk/input/postgresql/HstoreTest.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.postgresql; 2 | 3 | import java.nio.file.Path; 4 | 5 | import org.embulk.config.ConfigDiff; 6 | import org.embulk.config.ConfigSource; 7 | import org.embulk.formatter.csv.CsvFormatterPlugin; 8 | import org.embulk.input.file.LocalFileInputPlugin; 9 | import org.embulk.output.file.LocalFileOutputPlugin; 10 | import org.embulk.parser.csv.CsvParserPlugin; 11 | import org.embulk.spi.FileInputPlugin; 12 | import org.embulk.spi.FileOutputPlugin; 13 | import org.embulk.spi.FormatterPlugin; 14 | import org.embulk.spi.InputPlugin; 15 | import org.embulk.spi.ParserPlugin; 16 | import org.embulk.test.TestingEmbulk.RunResult; 17 | import org.embulk.test.EmbulkTests; 18 | import org.embulk.test.TestingEmbulk; 19 | import org.junit.Before; 20 | import org.junit.Rule; 21 | import org.junit.Test; 22 | 23 | import static org.embulk.input.postgresql.PostgreSQLTests.execute; 24 | import static org.embulk.test.EmbulkTests.readSortedFile; 25 | import static org.hamcrest.Matchers.is; 26 | import static org.junit.Assert.assertThat; 27 | 28 | public class HstoreTest 29 | { 30 | private static final String BASIC_RESOURCE_PATH = "/org/embulk/input/postgresql/test/expect/hstore/"; 31 | 32 | private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) 33 | { 34 | return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName); 35 | } 36 | 37 | private static String readResource(String fileName) 38 | { 39 | return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName); 40 | } 41 | 42 | @Rule 43 | public TestingEmbulk embulk = TestingEmbulk.builder() 44 | .registerPlugin(FileInputPlugin.class, "file", LocalFileInputPlugin.class) 45 | .registerPlugin(ParserPlugin.class, "csv", CsvParserPlugin.class) 46 | .registerPlugin(FormatterPlugin.class, "csv", CsvFormatterPlugin.class) 47 | .registerPlugin(FileOutputPlugin.class, "file", LocalFileOutputPlugin.class) 48 | .registerPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class) 49 | .build(); 50 | 51 | private ConfigSource baseConfig; 52 | 53 | @Before 54 | public void setup() 55 | { 56 | baseConfig = PostgreSQLTests.baseConfig(); 57 | } 58 | 59 | @Test 60 | public void loadAsStringByDefault() throws Exception 61 | { 62 | execute(readResource("setup.sql")); 63 | 64 | Path out1 = embulk.createTempFile("csv"); 65 | RunResult result1 = embulk.runInput( 66 | baseConfig.merge(loadYamlResource(embulk, "as_string.yml")), 67 | out1); 68 | assertThat( 69 | readSortedFile(out1), 70 | is(readResource("expected_string.csv"))); 71 | } 72 | 73 | @Test 74 | public void loadAsJson() throws Exception 75 | { 76 | execute(readResource("setup.sql")); 77 | 78 | Path out1 = embulk.createTempFile("csv"); 79 | RunResult result1 = embulk.runInput( 80 | baseConfig.merge(loadYamlResource(embulk, "as_json.yml")), 81 | out1); 82 | assertThat( 83 | readSortedFile(out1), 84 | is(readResource("expected_json.csv"))); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/java/org/embulk/input/postgresql/PostgreSQLTests.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.postgresql; 2 | 3 | import org.embulk.test.EmbulkTests; 4 | import com.google.common.base.Throwables; 5 | import com.google.common.io.ByteStreams; 6 | import java.io.IOException; 7 | import org.embulk.config.ConfigSource; 8 | import static java.util.Locale.ENGLISH; 9 | 10 | public class PostgreSQLTests 11 | { 12 | public static ConfigSource baseConfig() 13 | { 14 | return EmbulkTests.config("EMBULK_INPUT_POSTGRESQL_TEST_CONFIG"); 15 | } 16 | 17 | public static void execute(String sql) 18 | { 19 | ConfigSource config = baseConfig(); 20 | ProcessBuilder pb = new ProcessBuilder( 21 | "psql", "-w", 22 | "--set", "ON_ERROR_STOP=1", 23 | "--host", config.get(String.class, "host"), 24 | "--username", config.get(String.class, "user"), 25 | "--dbname", config.get(String.class, "database"), 26 | "-c", convert(sql)); 27 | pb.environment().put("PGPASSWORD", config.get(String.class, "password")); 28 | pb.redirectErrorStream(true); 29 | int code; 30 | try { 31 | Process process = pb.start(); 32 | ByteStreams.copy(process.getInputStream(), System.out); 33 | code = process.waitFor(); 34 | } catch (IOException | InterruptedException ex) { 35 | throw Throwables.propagate(ex); 36 | } 37 | if (code != 0) { 38 | throw new RuntimeException(String.format(ENGLISH, 39 | "Command finished with non-zero exit code. Exit code is %d.", code)); 40 | } 41 | } 42 | 43 | private static String convert(String sql) 44 | { 45 | if (System.getProperty("os.name").contains("Windows")) { 46 | // '"' should be '\"' and '\' should be '\\' in Windows 47 | return sql.replace("\\\\", "\\").replace("\\", "\\\\").replace("\"", "\\\""); 48 | } 49 | return sql; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/java/org/embulk/input/postgresql/UUIDTest.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.postgresql; 2 | 3 | import static org.embulk.input.postgresql.PostgreSQLTests.execute; 4 | import static org.embulk.test.EmbulkTests.readSortedFile; 5 | import static org.hamcrest.Matchers.is; 6 | import static org.junit.Assert.assertThat; 7 | 8 | import java.nio.file.Path; 9 | 10 | import org.embulk.config.ConfigSource; 11 | import org.embulk.formatter.csv.CsvFormatterPlugin; 12 | import org.embulk.input.file.LocalFileInputPlugin; 13 | import org.embulk.output.file.LocalFileOutputPlugin; 14 | import org.embulk.parser.csv.CsvParserPlugin; 15 | import org.embulk.spi.FileInputPlugin; 16 | import org.embulk.spi.FileOutputPlugin; 17 | import org.embulk.spi.FormatterPlugin; 18 | import org.embulk.spi.InputPlugin; 19 | import org.embulk.spi.ParserPlugin; 20 | import org.embulk.test.EmbulkTests; 21 | import org.embulk.test.TestingEmbulk; 22 | import org.embulk.test.TestingEmbulk.RunResult; 23 | import org.junit.Before; 24 | import org.junit.Rule; 25 | import org.junit.Test; 26 | 27 | public class UUIDTest 28 | { 29 | private static final String BASIC_RESOURCE_PATH = "/org/embulk/input/postgresql/test/expect/uuid/"; 30 | 31 | private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) 32 | { 33 | return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName); 34 | } 35 | 36 | private static String readResource(String fileName) 37 | { 38 | return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName); 39 | } 40 | 41 | @Rule 42 | public TestingEmbulk embulk = TestingEmbulk.builder() 43 | .registerPlugin(FileInputPlugin.class, "file", LocalFileInputPlugin.class) 44 | .registerPlugin(ParserPlugin.class, "csv", CsvParserPlugin.class) 45 | .registerPlugin(FormatterPlugin.class, "csv", CsvFormatterPlugin.class) 46 | .registerPlugin(FileOutputPlugin.class, "file", LocalFileOutputPlugin.class) 47 | .registerPlugin(InputPlugin.class, "postgresql", PostgreSQLInputPlugin.class) 48 | .build(); 49 | 50 | private ConfigSource baseConfig; 51 | 52 | @Before 53 | public void setup() 54 | { 55 | baseConfig = PostgreSQLTests.baseConfig(); 56 | } 57 | 58 | @Test 59 | public void loadAsStringByDefault() throws Exception 60 | { 61 | execute(readResource("setup.sql")); 62 | 63 | Path out1 = embulk.createTempFile("csv"); 64 | RunResult result1 = embulk.runInput( 65 | baseConfig.merge(loadYamlResource(embulk, "as_string.yml")), 66 | out1); 67 | assertThat( 68 | readSortedFile(out1), 69 | is(readResource("expected_string.csv"))); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/array/as_json.yml: -------------------------------------------------------------------------------- 1 | table: input_array 2 | column_options: 3 | c1: {type: json} 4 | c2: {type: json} 5 | c3: {type: json} 6 | c4: {type: json} 7 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/array/as_string.yml: -------------------------------------------------------------------------------- 1 | table: input_array 2 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/array/expected_json.csv: -------------------------------------------------------------------------------- 1 | "[1000,2000,3000,4000]","[[""red"",""green""],[""blue"",""cyan""]]",[[[true]]],[1.23456789E9] 2 | "[5000,6000,7000,8000]","[[""yellow"",""magenta""],[""purple"",""light,dark""]]","[[[true,true],[false,false]],[[true,false],[false,true]]]",[1.2345678901234567E19] 3 | [1000],"[[""\"""",""{\\}"",""{a,b}""]]",[true],[1.2345678901234567E19] 4 | [],[],[],[] 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/array/expected_string.csv: -------------------------------------------------------------------------------- 1 | "{1000,2000,3000,4000}","{{red,green},{blue,cyan}}",{{{t}}},{1234567890} 2 | "{5000,6000,7000,8000}","{{yellow,magenta},{purple,""light,dark""}}","{{{t,t},{f,f}},{{t,f},{f,t}}}",{12345678901234567890} 3 | {1000},"{{""\"""",""{\\}"",""{a,b}""}}",{t},{12345678901234567890.1234567890} 4 | {},{},{},{} 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/array/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists input_array; 2 | 3 | create table input_array ( 4 | c1 integer[], 5 | c2 text[][], 6 | c3 bool[][][], 7 | c4 decimal[] 8 | ); 9 | 10 | insert into input_array (c1, c2, c3, c4) values ('{1000, 2000, 3000, 4000}', '{{"red", "green"}, {"blue", "cyan"}}', '{{{true}}}', '{1234567890}'); 11 | 12 | insert into input_array (c1, c2, c3, c4) values ('{5000, 6000, 7000, 8000}', '{{"yellow", "magenta"}, {"purple", "light,dark"}}', '{{{t,t},{f,f}},{{t,f},{f,t}}}', '{12345678901234567890}'); 13 | 14 | insert into input_array (c1, c2, c3, c4) values ('{1000}', '{{"\"", "{\\}", "{a,b}"}}', '{true}', '{12345678901234567890.1234567890}'); 15 | 16 | insert into input_array (c1, c2, c3, c4) values ('{}', '{}', '{}', '{}'); 17 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_json.yml: -------------------------------------------------------------------------------- 1 | table: input_hstore 2 | column_options: 3 | c1: {type: json} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/as_string.yml: -------------------------------------------------------------------------------- 1 | table: input_hstore 2 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_json.csv: -------------------------------------------------------------------------------- 1 | "{""a"":""b""}" 2 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/expected_string.csv: -------------------------------------------------------------------------------- 1 | """a""=>""b""" 2 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/hstore/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists input_hstore; 2 | 3 | create extension if not exists hstore; 4 | 5 | create table input_hstore ( 6 | c1 hstore 7 | ); 8 | 9 | insert into input_hstore (c1) values 10 | ('"a" => "b"') 11 | ; 12 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/char/config_1.yml: -------------------------------------------------------------------------------- 1 | table: char_load 2 | incremental: true 3 | incremental_columns: [name] 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/char/config_2.yml: -------------------------------------------------------------------------------- 1 | table: char_load 2 | last_record: ['A4'] 3 | incremental: true 4 | incremental_columns: [name] 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/char/expected_1.csv: -------------------------------------------------------------------------------- 1 | A1,first 2 | A2,first 3 | A3,first 4 | A4,first 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/char/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['A4'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/char/expected_2.csv: -------------------------------------------------------------------------------- 1 | A5,more_load 2 | A9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/char/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['A9'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/char/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into char_load (name, note) values 3 | ('A0', 'more_skip'), 4 | ('A4', 'more_skip'), 5 | ('A9', 'more_load'), 6 | ('A5', 'more_load'); 7 | 8 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/char/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists char_load; 2 | 3 | create table char_load ( 4 | name char(2) not null, 5 | note text 6 | ); 7 | 8 | insert into char_load (name, note) values 9 | ('A3', 'first'), 10 | ('A4', 'first'), 11 | ('A2', 'first'), 12 | ('A1', 'first'); 13 | 14 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_1.yml: -------------------------------------------------------------------------------- 1 | table: int_load 2 | incremental: true 3 | incremental_columns: [num] 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/config_2.yml: -------------------------------------------------------------------------------- 1 | table: int_load 2 | last_record: [4] 3 | incremental: true 4 | incremental_columns: [num] 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,first 2 | 2,first 3 | 3,first 4 | 4,first 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,more_load 2 | 9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into int_load (num, note) values 3 | (0, 'more_skip'), 4 | (4, 'more_skip'), 5 | (9, 'more_load'), 6 | (5, 'more_load'); 7 | 8 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/int/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists int_load; 2 | 3 | create table int_load ( 4 | num int not null, 5 | note text 6 | ); 7 | 8 | insert into int_load (num, note) values 9 | (3, 'first'), 10 | (4, 'first'), 11 | (2, 'first'), 12 | (1, 'first'); 13 | 14 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0] 2 | incremental: true 3 | incremental_columns: [num] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | num,note 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | ORDER BY 14 | num ASC 15 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4] 2 | incremental: true 3 | incremental_columns: [num] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | num,note 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | ORDER BY 14 | num ASC 15 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,first 2 | 2,first 3 | 3,first 4 | 4,first 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,more_load 2 | 9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into query_load (num, num2, note) values 3 | (0, 100, 'more_skip'), 4 | (4, 104, 'more_skip'), 5 | (9, 109, 'more_load'), 6 | (5, 105, 'more_load'); 7 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0,0] 2 | incremental: true 3 | incremental_columns: [num,num2] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | * 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | OR (num = :num AND num2 > :num2) 14 | ORDER BY 15 | num ASC, 16 | num2 ASC 17 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4,104] 2 | incremental: true 3 | incremental_columns: [num,num2] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | * 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | OR (num = :num AND num2 > :num2) 14 | ORDER BY 15 | num ASC, 16 | num2 ASC 17 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,101,first 2 | 2,102,first 3 | 3,103,first 4 | 4,104,first 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4,104] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,105,more_load 2 | 9,109,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/multi_columns_expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9,109] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/query/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists query_load; 2 | 3 | create table query_load ( 4 | num int not null, 5 | num2 int not null, 6 | note text 7 | ); 8 | 9 | insert into query_load (num, num2, note) values 10 | (3, 103, 'first'), 11 | (4, 104, 'first'), 12 | (2, 102, 'first'), 13 | (1, 101, 'first'); 14 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_1.yml: -------------------------------------------------------------------------------- 1 | table: load 2 | default_time_zone: +0300 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/config_2.yml: -------------------------------------------------------------------------------- 1 | table: load 2 | last_record: ['2016-11-02T04:00:05.333003'] 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.csv: -------------------------------------------------------------------------------- 1 | 2016-11-01 23:00:01.000000 +0000,first 2 | 2016-11-02 00:00:02.000000 +0000,first 3 | 2016-11-02 01:00:03.000000 +0000,first 4 | 2016-11-02 02:00:04.000000 +0000,first 5 | 2016-11-02 02:00:05.111001 +0000,first 6 | 2016-11-02 02:00:05.222002 +0000,first 7 | 2016-11-02 02:00:05.333003 +0000,first 8 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:05.333003'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 02:00:05.333004 +0000,more_load 2 | 2016-11-02 02:00:06.000000 +0000,more_load 3 | 2016-11-02 02:00:06.000000 +0000,more_load 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:06.000000'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into load (time, note) values 3 | ('2016-11-02 04:00:00', 'more_skip'), 4 | ('2016-11-02 04:00:05.333000', 'more_skip'), 5 | ('2016-11-02 04:00:05.333003', 'more_skip'), 6 | ('2016-11-02 04:00:05.333004', 'more_load'), 7 | ('2016-11-02 04:00:06', 'more_load'), 8 | ('2016-11-02 04:00:06', 'more_load'); 9 | 10 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamp/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists load; 2 | 3 | create table load ( 4 | time timestamp without time zone not null, 5 | note text 6 | ); 7 | 8 | insert into load (time, note) values 9 | ('2016-11-02 01:00:01', 'first'), 10 | ('2016-11-02 02:00:02', 'first'), 11 | ('2016-11-02 03:00:03', 'first'), 12 | ('2016-11-02 04:00:04', 'first'), 13 | ('2016-11-02 04:00:05.111001', 'first'), 14 | ('2016-11-02 04:00:05.222002', 'first'), 15 | ('2016-11-02 04:00:05.333003', 'first'); 16 | 17 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_1.yml: -------------------------------------------------------------------------------- 1 | table: load 2 | default_time_zone: +0300 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/config_2.yml: -------------------------------------------------------------------------------- 1 | table: load 2 | last_record: ['2016-11-02T04:00:05.333003Z'] 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 01:00:01.000000 +0000,first 2 | 2016-11-02 02:00:02.000000 +0000,first 3 | 2016-11-02 03:00:03.000000 +0000,first 4 | 2016-11-02 04:00:04.000000 +0000,first 5 | 2016-11-02 04:00:05.111001 +0000,first 6 | 2016-11-02 04:00:05.222002 +0000,first 7 | 2016-11-02 04:00:05.333003 +0000,first 8 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:05.333003Z'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 04:00:05.333004 +0000,more_load 2 | 2016-11-02 04:00:06.000000 +0000,more_load 3 | 2016-11-02 04:00:06.000000 +0000,more_load 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:06.000000Z'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into load (time, note) values 3 | ('2016-11-02 04:00:00+0000', 'more_skip'), 4 | ('2016-11-02 04:00:05.333000+0000', 'more_skip'), 5 | ('2016-11-02 04:00:05.333003+0000', 'more_skip'), 6 | ('2016-11-02 04:00:05.333004+0000', 'more_load'), 7 | ('2016-11-02 04:00:06+0000', 'more_load'), 8 | ('2016-11-02 04:00:06+0000', 'more_load'); 9 | 10 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/incremental/timestamptz/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists load; 2 | 3 | create table load ( 4 | time timestamptz(6) not null, 5 | note text 6 | ); 7 | 8 | insert into load (time, note) values 9 | ('2016-11-02 01:00:01+0000', 'first'), 10 | ('2016-11-02 02:00:02+0000', 'first'), 11 | ('2016-11-02 03:00:03+0000', 'first'), 12 | ('2016-11-02 04:00:04+0000', 'first'), 13 | ('2016-11-02 04:00:05.111001+0000', 'first'), 14 | ('2016-11-02 04:00:05.222002+0000', 'first'), 15 | ('2016-11-02 04:00:05.333003+0000', 'first'); 16 | 17 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/uuid/as_string.yml: -------------------------------------------------------------------------------- 1 | table: input_uuid 2 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/uuid/expected_string.csv: -------------------------------------------------------------------------------- 1 | d29977bb-55fb-4a9d-9e3b-97f52b191f49 2 | -------------------------------------------------------------------------------- /embulk-input-postgresql/src/test/resources/org/embulk/input/postgresql/test/expect/uuid/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists input_uuid; 2 | 3 | create table input_uuid ( 4 | c1 uuid 5 | ); 6 | 7 | insert into input_uuid (c1) values 8 | ('d29977bb-55fb-4a9d-9e3b-97f52b191f49') 9 | ; 10 | -------------------------------------------------------------------------------- /embulk-input-redshift/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation(project(path: ":embulk-input-jdbc", configuration: "runtimeElements")) 3 | implementation(project(path: ":embulk-input-postgresql", configuration: "runtimeElements")) 4 | implementation "org.postgresql:postgresql:9.4-1205-jdbc41" 5 | 6 | testImplementation project(":embulk-input-jdbc").sourceSets.test.output 7 | 8 | testImplementation "com.google.guava:guava:18.0" 9 | testImplementation "org.embulk:embulk-formatter-csv:0.11.1" 10 | testImplementation "org.embulk:embulk-input-file:0.11.0" 11 | testImplementation "org.embulk:embulk-output-file:0.11.0" 12 | testImplementation "org.embulk:embulk-parser-csv:0.11.3" 13 | } 14 | 15 | embulkPlugin { 16 | mainClass = "org.embulk.input.redshift.RedshiftInputPlugin" 17 | category = "input" 18 | type = "redshift" 19 | } 20 | 21 | publishing { 22 | publications { 23 | maven(MavenPublication) { 24 | pom { // https://central.sonatype.org/pages/requirements.html 25 | developers { 26 | developer { 27 | name = "Sadayuki Furuhashi" 28 | email = "frsyuki@gmail.com" 29 | } 30 | developer { 31 | name = "Hitoshi Tanaka" 32 | email = "thitoshi@cac.co.jp" 33 | } 34 | developer { 35 | name = "Satoshi Akama" 36 | email = "satoshiakama@gmail.com" 37 | } 38 | developer { 39 | name = "Dai MIKURUBE" 40 | email = "dmikurube@treasure-data.com" 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /embulk-input-redshift/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 9 | org.embulk:embulk-spi:0.11=compileClasspath 10 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 11 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 12 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 13 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 14 | org.msgpack:msgpack-core:0.8.24=compileClasspath 15 | org.postgresql:postgresql:9.4-1205-jdbc41=compileClasspath,runtimeClasspath 16 | org.slf4j:slf4j-api:2.0.7=compileClasspath 17 | empty= 18 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/main/java/org/embulk/input/redshift/getter/RedshiftColumnGetterFactory.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.redshift.getter; 2 | 3 | import java.time.ZoneId; 4 | import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask; 5 | import org.embulk.input.jdbc.JdbcColumn; 6 | import org.embulk.input.jdbc.JdbcColumnOption; 7 | import org.embulk.input.jdbc.JdbcInputConnection; 8 | import org.embulk.input.jdbc.getter.ColumnGetter; 9 | import org.embulk.input.jdbc.getter.ColumnGetterFactory; 10 | import org.embulk.input.jdbc.getter.TimestampWithTimeZoneIncrementalHandler; 11 | import org.embulk.input.jdbc.getter.TimestampWithoutTimeZoneIncrementalHandler; 12 | import org.embulk.spi.PageBuilder; 13 | 14 | public class RedshiftColumnGetterFactory extends ColumnGetterFactory 15 | { 16 | public RedshiftColumnGetterFactory(final PageBuilder to, final ZoneId defaultTimeZone) 17 | { 18 | super(to, defaultTimeZone); 19 | } 20 | 21 | @Override 22 | public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option) 23 | { 24 | ColumnGetter getter = super.newColumnGetter(con, task, column, option); 25 | 26 | // incremental loading wrapper 27 | switch (column.getTypeName()) { 28 | case "timestamptz": 29 | return new TimestampWithTimeZoneIncrementalHandler(getter); 30 | case "timestamp": 31 | return new TimestampWithoutTimeZoneIncrementalHandler(getter); 32 | default: 33 | return getter; 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/java/org/embulk/input/redshift/RedshiftTests.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.redshift; 2 | 3 | import com.google.common.base.Throwables; 4 | import com.google.common.io.ByteStreams; 5 | import org.embulk.config.ConfigSource; 6 | import org.embulk.test.EmbulkTests; 7 | 8 | import java.io.IOException; 9 | 10 | import static java.util.Locale.ENGLISH; 11 | 12 | public class RedshiftTests 13 | { 14 | public static ConfigSource baseConfig() 15 | { 16 | return EmbulkTests.config("EMBULK_INPUT_REDSHIFT_TEST_CONFIG"); 17 | } 18 | 19 | public static void execute(String sql) 20 | { 21 | ConfigSource config = baseConfig(); 22 | ProcessBuilder pb = new ProcessBuilder("psql", "-w", "--set", "ON_ERROR_STOP=1", "-c", sql); 23 | pb.environment().put("PGUSER", config.get(String.class, "user")); 24 | pb.environment().put("PGPASSWORD", config.get(String.class, "password")); 25 | pb.environment().put("PGDATABASE", config.get(String.class, "database")); 26 | pb.environment().put("PGHOST", config.get(String.class, "host", "localhost")); 27 | pb.environment().put("PGPORT", config.get(String.class, "port", "5439")); 28 | pb.redirectErrorStream(true); 29 | 30 | int code; 31 | try { 32 | Process process = pb.start(); 33 | ByteStreams.copy(process.getInputStream(), System.out); 34 | code = process.waitFor(); 35 | } catch (IOException | InterruptedException ex) { 36 | throw Throwables.propagate(ex); 37 | } 38 | if (code != 0) { 39 | throw new RuntimeException(String.format(ENGLISH, 40 | "Command finished with non-zero exit code. Exit code is %d.", code)); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/config_1.yml: -------------------------------------------------------------------------------- 1 | table: int_load 2 | incremental: true 3 | incremental_columns: [name] 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/config_2.yml: -------------------------------------------------------------------------------- 1 | table: int_load 2 | last_record: ['A4'] 3 | incremental: true 4 | incremental_columns: [name] 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_1.csv: -------------------------------------------------------------------------------- 1 | A1,first 2 | A2,first 3 | A3,first 4 | A4,first 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['A4'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_2.csv: -------------------------------------------------------------------------------- 1 | A5,more_load 2 | A9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['A9'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into int_load (name, note) values 3 | ('A0', 'more_skip'), 4 | ('A4', 'more_skip'), 5 | ('A9', 'more_load'), 6 | ('A5', 'more_load'); 7 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/char/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists int_load; 2 | 3 | create table int_load ( 4 | name char(2) not null, 5 | note text 6 | ); 7 | 8 | insert into int_load (name, note) values 9 | ('A3', 'first'), 10 | ('A4', 'first'), 11 | ('A2', 'first'), 12 | ('A1', 'first'); 13 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/config_1.yml: -------------------------------------------------------------------------------- 1 | table: int_load 2 | incremental: true 3 | incremental_columns: [num] 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/config_2.yml: -------------------------------------------------------------------------------- 1 | table: int_load 2 | last_record: [4] 3 | incremental: true 4 | incremental_columns: [num] 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,first 2 | 2,first 3 | 3,first 4 | 4,first 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,more_load 2 | 9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into int_load (num, note) values 3 | (0, 'more_skip'), 4 | (4, 'more_skip'), 5 | (9, 'more_load'), 6 | (5, 'more_load'); 7 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/int/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists int_load; 2 | 3 | create table int_load ( 4 | num int not null, 5 | note text 6 | ); 7 | 8 | insert into int_load (num, note) values 9 | (3, 'first'), 10 | (4, 'first'), 11 | (2, 'first'), 12 | (1, 'first'); 13 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0] 2 | incremental: true 3 | incremental_columns: [num] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | num,note 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | ORDER BY 14 | num ASC 15 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4] 2 | incremental: true 3 | incremental_columns: [num] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | num,note 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | ORDER BY 14 | num ASC 15 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,first 2 | 2,first 3 | 3,first 4 | 4,first 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,more_load 2 | 9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into query_load (num, num2, note) values 3 | (0, 100, 'more_skip'), 4 | (4, 104, 'more_skip'), 5 | (9, 109, 'more_load'), 6 | (5, 105, 'more_load'); 7 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0,0] 2 | incremental: true 3 | incremental_columns: [num,num2] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | * 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | OR (num = :num AND num2 > :num2) 14 | ORDER BY 15 | num ASC, 16 | num2 ASC 17 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4,104] 2 | incremental: true 3 | incremental_columns: [num,num2] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | * 8 | FROM 9 | query_load 10 | WHERE 11 | num IS NOT NULL 12 | AND num > :num 13 | OR (num = :num AND num2 > :num2) 14 | ORDER BY 15 | num ASC, 16 | num2 ASC 17 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,101,first 2 | 2,102,first 3 | 3,103,first 4 | 4,104,first 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4,104] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,105,more_load 2 | 9,109,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/multi_columns_expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9,109] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/query/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists query_load; 2 | 3 | create table query_load ( 4 | num int not null, 5 | num2 int not null, 6 | note text 7 | ); 8 | 9 | insert into query_load (num, num2, note) values 10 | (3, 103, 'first'), 11 | (4, 104, 'first'), 12 | (2, 102, 'first'), 13 | (1, 101, 'first'); 14 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_1.yml: -------------------------------------------------------------------------------- 1 | table: load 2 | default_time_zone: +0300 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/config_2.yml: -------------------------------------------------------------------------------- 1 | table: load 2 | last_record: ['2016-11-02T04:00:05.333003'] 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_1.csv: -------------------------------------------------------------------------------- 1 | 2016-11-01 23:00:01.000000 +0000,first 2 | 2016-11-02 00:00:02.000000 +0000,first 3 | 2016-11-02 01:00:03.000000 +0000,first 4 | 2016-11-02 02:00:04.000000 +0000,first 5 | 2016-11-02 02:00:05.111001 +0000,first 6 | 2016-11-02 02:00:05.222002 +0000,first 7 | 2016-11-02 02:00:05.333003 +0000,first 8 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:05.333003'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_2.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 02:00:05.333004 +0000,more_load 2 | 2016-11-02 02:00:06.000000 +0000,more_load 3 | 2016-11-02 02:00:06.000000 +0000,more_load 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:06.000000'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into load (time, note) values 3 | ('2016-11-02 04:00:00', 'more_skip'), 4 | ('2016-11-02 04:00:05.333000', 'more_skip'), 5 | ('2016-11-02 04:00:05.333003', 'more_skip'), 6 | ('2016-11-02 04:00:05.333004', 'more_load'), 7 | ('2016-11-02 04:00:06', 'more_load'), 8 | ('2016-11-02 04:00:06', 'more_load'); 9 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamp/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists load; 2 | 3 | create table load ( 4 | time timestamp without time zone not null, 5 | note text 6 | ); 7 | 8 | insert into load (time, note) values 9 | ('2016-11-02 01:00:01', 'first'), 10 | ('2016-11-02 02:00:02', 'first'), 11 | ('2016-11-02 03:00:03', 'first'), 12 | ('2016-11-02 04:00:04', 'first'), 13 | ('2016-11-02 04:00:05.111001', 'first'), 14 | ('2016-11-02 04:00:05.222002', 'first'), 15 | ('2016-11-02 04:00:05.333003', 'first'); 16 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/config_1.yml: -------------------------------------------------------------------------------- 1 | table: load 2 | default_time_zone: +0300 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/config_2.yml: -------------------------------------------------------------------------------- 1 | table: load 2 | last_record: ['2016-11-02T04:00:05.333003Z'] 3 | incremental: true 4 | incremental_columns: [time] 5 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_1.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 01:00:01.000000 +0000,first 2 | 2016-11-02 02:00:02.000000 +0000,first 3 | 2016-11-02 03:00:03.000000 +0000,first 4 | 2016-11-02 04:00:04.000000 +0000,first 5 | 2016-11-02 04:00:05.111001 +0000,first 6 | 2016-11-02 04:00:05.222002 +0000,first 7 | 2016-11-02 04:00:05.333003 +0000,first 8 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:05.333003Z'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_2.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 04:00:05.333004 +0000,more_load 2 | 2016-11-02 04:00:06.000000 +0000,more_load 3 | 2016-11-02 04:00:06.000000 +0000,more_load 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02T04:00:06.000000Z'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | insert into load (time, note) values 3 | ('2016-11-02 04:00:00+0000', 'more_skip'), 4 | ('2016-11-02 04:00:05.333000+0000', 'more_skip'), 5 | ('2016-11-02 04:00:05.333003+0000', 'more_skip'), 6 | ('2016-11-02 04:00:05.333004+0000', 'more_load'), 7 | ('2016-11-02 04:00:06+0000', 'more_load'), 8 | ('2016-11-02 04:00:06+0000', 'more_load'); 9 | -------------------------------------------------------------------------------- /embulk-input-redshift/src/test/resources/org/embulk/input/redshift/test/expect/incremental/timestamptz/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists load; 2 | 3 | create table load ( 4 | time timestamptz not null, 5 | note text 6 | ); 7 | 8 | insert into load (time, note) values 9 | ('2016-11-02 01:00:01+0000', 'first'), 10 | ('2016-11-02 02:00:02+0000', 'first'), 11 | ('2016-11-02 03:00:03+0000', 'first'), 12 | ('2016-11-02 04:00:04+0000', 'first'), 13 | ('2016-11-02 04:00:05.111001+0000', 'first'), 14 | ('2016-11-02 04:00:05.222002+0000', 'first'), 15 | ('2016-11-02 04:00:05.333003+0000', 'first'); 16 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation(project(path: ":embulk-input-jdbc", configuration: "runtimeElements")) 3 | implementation "com.microsoft.sqlserver:mssql-jdbc:7.2.2.jre8" 4 | implementation("com.microsoft.azure:adal4j:1.6.7") { 5 | exclude group: 'org.slf4j', module: 'slf4j-api' 6 | exclude group: "org.apache.commons", module: "commons-lang3" 7 | } 8 | implementation "net.sourceforge.jtds:jtds:1.3.1" 9 | implementation "org.apache.commons:commons-lang3:3.4" 10 | 11 | testImplementation "com.google.guava:guava:18.0" 12 | testImplementation "org.embulk:embulk-formatter-csv:0.11.1" 13 | testImplementation "org.embulk:embulk-input-file:0.11.0" 14 | testImplementation "org.embulk:embulk-output-file:0.11.0" 15 | testImplementation "org.embulk:embulk-parser-csv:0.11.3" 16 | } 17 | 18 | embulkPlugin { 19 | mainClass = "org.embulk.input.sqlserver.SQLServerInputPlugin" 20 | category = "input" 21 | type = "sqlserver" 22 | } 23 | 24 | publishing { 25 | publications { 26 | maven(MavenPublication) { 27 | pom { // https://central.sonatype.org/pages/requirements.html 28 | developers { 29 | developer { 30 | name = "Hitoshi Tanaka" 31 | email = "thitoshi@cac.co.jp" 32 | } 33 | developer { 34 | name = "Sadayuki Furuhashi" 35 | email = "frsyuki@gmail.com" 36 | } 37 | developer { 38 | name = "Muga Nishizawa" 39 | email = "muga.nishizawa@gmail.com" 40 | } 41 | developer { 42 | name = "Shintaro Kimura" 43 | email = "kmrshntr@gmail.com" 44 | } 45 | developer { 46 | name = "Hieu Duong" 47 | email = "duongminhhieu89@gmail.com" 48 | } 49 | developer { 50 | name = "Dai MIKURUBE" 51 | email = "dmikurube@treasure-data.com" 52 | } 53 | } 54 | } 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | com.github.stephenc.jcip:jcip-annotations:1.0-1=compileClasspath,runtimeClasspath 9 | com.google.code.gson:gson:2.8.0=compileClasspath,runtimeClasspath 10 | com.microsoft.azure:adal4j:1.6.7=compileClasspath,runtimeClasspath 11 | com.microsoft.sqlserver:mssql-jdbc:7.2.2.jre8=compileClasspath,runtimeClasspath 12 | com.nimbusds:content-type:2.1=compileClasspath,runtimeClasspath 13 | com.nimbusds:lang-tag:1.5=compileClasspath,runtimeClasspath 14 | com.nimbusds:nimbus-jose-jwt:9.8.1=compileClasspath,runtimeClasspath 15 | com.nimbusds:oauth2-oidc-sdk:9.4=compileClasspath,runtimeClasspath 16 | commons-codec:commons-codec:1.14=compileClasspath,runtimeClasspath 17 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 18 | net.minidev:accessors-smart:2.4.2=compileClasspath,runtimeClasspath 19 | net.minidev:json-smart:2.4.2=compileClasspath,runtimeClasspath 20 | net.sourceforge.jtds:jtds:1.3.1=compileClasspath,runtimeClasspath 21 | org.apache.commons:commons-lang3:3.4=compileClasspath,runtimeClasspath 22 | org.embulk:embulk-spi:0.11=compileClasspath 23 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 24 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 25 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 26 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 27 | org.msgpack:msgpack-core:0.8.24=compileClasspath 28 | org.ow2.asm:asm:8.0.1=compileClasspath,runtimeClasspath 29 | org.slf4j:slf4j-api:2.0.7=compileClasspath 30 | empty= 31 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/main/java/org/embulk/input/sqlserver/SQLServerInputConnection.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.sqlserver; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | import org.embulk.input.jdbc.JdbcInputConnection; 6 | 7 | public class SQLServerInputConnection extends JdbcInputConnection 8 | { 9 | private String transactionIsolationLevel; 10 | 11 | public SQLServerInputConnection(Connection connection, String schemaName) throws SQLException 12 | { 13 | super(connection, schemaName); 14 | } 15 | 16 | public SQLServerInputConnection(Connection connection, String schemaName, String transactionIsolationLevel) 17 | throws SQLException 18 | { 19 | super(connection, schemaName); 20 | this.transactionIsolationLevel = transactionIsolationLevel; 21 | } 22 | 23 | @Override 24 | protected void setSearchPath(String schema) throws SQLException 25 | { 26 | // NOP 27 | } 28 | 29 | @Override 30 | protected String buildTableName(String tableName) 31 | { 32 | StringBuilder sb = new StringBuilder(); 33 | if (schemaName != null) { 34 | sb.append(quoteIdentifierString(schemaName)).append("."); 35 | } 36 | sb.append(quoteIdentifierString(tableName)); 37 | if (transactionIsolationLevel != null) { 38 | sb.append(" with (" + transactionIsolationLevel + ")"); 39 | } 40 | return sb.toString(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/main/java/org/embulk/input/sqlserver/getter/SQLServerColumnGetterFactory.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.sqlserver.getter; 2 | 3 | import java.time.ZoneId; 4 | import org.embulk.input.jdbc.AbstractJdbcInputPlugin; 5 | import org.embulk.input.jdbc.JdbcColumn; 6 | import org.embulk.input.jdbc.JdbcColumnOption; 7 | import org.embulk.input.jdbc.JdbcInputConnection; 8 | import org.embulk.input.jdbc.getter.*; 9 | import org.embulk.spi.PageBuilder; 10 | 11 | public class SQLServerColumnGetterFactory extends ColumnGetterFactory 12 | { 13 | public SQLServerColumnGetterFactory(final PageBuilder to, final ZoneId defaultTimeZone) 14 | { 15 | super(to, defaultTimeZone); 16 | } 17 | 18 | @Override 19 | public ColumnGetter newColumnGetter(JdbcInputConnection con, AbstractJdbcInputPlugin.PluginTask task, JdbcColumn column, JdbcColumnOption option) 20 | { 21 | switch (column.getTypeName()) { 22 | case "date": 23 | case "datetime2": 24 | case "time": 25 | case "sql_variant": 26 | // DateTimeOffset is available only in MSSQL 27 | case "datetimeoffset": 28 | // because jTDS driver, default JDBC driver for older embulk-input-sqlserver, returns Types.VARCHAR as JDBC type for these types. 29 | return new StringColumnGetter(to, getToType(option)); 30 | 31 | case "datetime": 32 | ColumnGetter getter = super.newColumnGetter(con, task, column, option); 33 | return new TimestampWithoutTimeZoneIncrementalHandler(getter); 34 | 35 | default: 36 | return super.newColumnGetter(con, task, column, option); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/java/org/embulk/input/sqlserver/BasicTest.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.sqlserver; 2 | 3 | import static org.embulk.input.sqlserver.SQLServerTests.execute; 4 | import static org.embulk.test.EmbulkTests.readSortedFile; 5 | import static org.hamcrest.Matchers.is; 6 | import static org.junit.Assert.assertThat; 7 | 8 | import java.nio.file.Path; 9 | 10 | import org.embulk.config.ConfigDiff; 11 | import org.embulk.config.ConfigSource; 12 | import org.embulk.formatter.csv.CsvFormatterPlugin; 13 | import org.embulk.input.file.LocalFileInputPlugin; 14 | import org.embulk.output.file.LocalFileOutputPlugin; 15 | import org.embulk.parser.csv.CsvParserPlugin; 16 | import org.embulk.spi.FileInputPlugin; 17 | import org.embulk.spi.FileOutputPlugin; 18 | import org.embulk.spi.FormatterPlugin; 19 | import org.embulk.spi.InputPlugin; 20 | import org.embulk.spi.ParserPlugin; 21 | import org.embulk.test.EmbulkTests; 22 | import org.embulk.test.TestingEmbulk; 23 | import org.junit.Before; 24 | import org.junit.Rule; 25 | import org.junit.Test; 26 | 27 | public class BasicTest 28 | { 29 | private static final String BASIC_RESOURCE_PATH = "/org/embulk/input/sqlserver/test/expect/basic/"; 30 | 31 | private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) 32 | { 33 | return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName); 34 | } 35 | 36 | private static String readResource(String fileName) 37 | { 38 | return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName); 39 | } 40 | 41 | @Rule 42 | public TestingEmbulk embulk = TestingEmbulk.builder() 43 | .registerPlugin(FileInputPlugin.class, "file", LocalFileInputPlugin.class) 44 | .registerPlugin(ParserPlugin.class, "csv", CsvParserPlugin.class) 45 | .registerPlugin(FormatterPlugin.class, "csv", CsvFormatterPlugin.class) 46 | .registerPlugin(FileOutputPlugin.class, "file", LocalFileOutputPlugin.class) 47 | .registerPlugin(InputPlugin.class, "sqlserver", SQLServerInputPlugin.class) 48 | .build(); 49 | 50 | private ConfigSource baseConfig; 51 | 52 | @Before 53 | public void setup() 54 | { 55 | baseConfig = SQLServerTests.baseConfig(); 56 | execute(readResource("setup.sql")); // setup rows 57 | } 58 | 59 | @Test 60 | public void test() throws Exception 61 | { 62 | Path out1 = embulk.createTempFile("csv"); 63 | TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_config.yml")), out1); 64 | assertThat(readSortedFile(out1), is(readResource("test_expected.csv"))); 65 | assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff"))); 66 | } 67 | 68 | @Test 69 | public void testJTDS() throws Exception 70 | { 71 | Path out1 = embulk.createTempFile("csv"); 72 | TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "jtds_config.yml")), out1); 73 | assertThat(readSortedFile(out1), is(readResource("test_expected.csv"))); 74 | assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff"))); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/java/org/embulk/input/sqlserver/SQLServerTests.java: -------------------------------------------------------------------------------- 1 | package org.embulk.input.sqlserver; 2 | 3 | import static java.util.Locale.ENGLISH; 4 | 5 | import java.io.IOException; 6 | import java.nio.charset.Charset; 7 | import java.nio.file.Files; 8 | import java.nio.file.Path; 9 | import java.util.ArrayList; 10 | import java.util.Arrays; 11 | import java.util.Collections; 12 | import java.util.List; 13 | 14 | import org.embulk.config.ConfigSource; 15 | import org.embulk.test.EmbulkTests; 16 | import org.embulk.test.TestingEmbulk; 17 | 18 | import com.google.common.base.Throwables; 19 | import com.google.common.collect.ImmutableList; 20 | import com.google.common.io.ByteStreams; 21 | 22 | public class SQLServerTests 23 | { 24 | private SQLServerTests() 25 | { 26 | // No instantiation. 27 | } 28 | 29 | public static ConfigSource baseConfig() 30 | { 31 | return EmbulkTests.config("EMBULK_INPUT_SQLSERVER_TEST_CONFIG"); 32 | } 33 | 34 | public static void execute(String sql, String... options) 35 | { 36 | ConfigSource config = baseConfig(); 37 | 38 | final ArrayList args = new ArrayList<>(); 39 | 40 | final String sqlcmdCommand = System.getenv("EMBULK_INPUT_SQLSERVER_TEST_SQLCMD_COMMAND"); 41 | if (sqlcmdCommand == null || sqlcmdCommand.isEmpty()) { 42 | args.add("sqlcmd"); 43 | } 44 | else { 45 | args.addAll(Arrays.asList(sqlcmdCommand.split(" "))); 46 | } 47 | 48 | args.add("-U"); 49 | args.add(config.get(String.class, "user")); 50 | args.add("-P"); 51 | args.add(config.get(String.class, "password")); 52 | args.add("-H"); 53 | args.add(config.get(String.class, "host")); 54 | args.add("-d"); 55 | args.add(config.get(String.class, "database")); 56 | args.add("-Q"); 57 | args.add(sql); 58 | for (String option : options) { 59 | args.add(option); 60 | } 61 | 62 | ProcessBuilder pb = new ProcessBuilder(args); 63 | pb.redirectErrorStream(true); 64 | int code; 65 | try { 66 | Process process = pb.start(); 67 | ByteStreams.copy(process.getInputStream(), System.out); 68 | code = process.waitFor(); 69 | } 70 | catch (IOException | InterruptedException ex) { 71 | throw Throwables.propagate(ex); 72 | } 73 | if (code != 0) { 74 | throw new RuntimeException(String.format(ENGLISH, 75 | "Command finished with non-zero exit code. Exit code is %d.", code)); 76 | } 77 | } 78 | 79 | public static String selectRecords(TestingEmbulk embulk, String tableName) throws IOException 80 | { 81 | Path temp = embulk.createTempFile("txt"); 82 | Files.delete(temp); 83 | 84 | // should not use UTF8 because of BOM 85 | execute("SET NOCOUNT ON; SELECT * FROM " + tableName, "-h", "-1", "-s", ",", "-W", "-f", "932", "-o", temp.toString()); 86 | 87 | List lines = Files.readAllLines(temp, Charset.forName("MS932")); 88 | Collections.sort(lines); 89 | StringBuilder sb = new StringBuilder(); 90 | for (String line : lines) { 91 | sb.append(line); 92 | sb.append("\n"); 93 | } 94 | return sb.toString(); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/basic/jtds_config.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | order_by: 'ID' 3 | driver_type: jtds 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/basic/setup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE TEST1 2 | CREATE TABLE TEST1 ( 3 | ID CHAR(4), 4 | TINYINT_ITEM TINYINT, 5 | SMALLINT_ITEM SMALLINT, 6 | INT_ITEM INT, 7 | BIGINT_ITEM BIGINT, 8 | BIT_ITEM BIT, 9 | DECIMAL_ITEM DECIMAL(12,2), 10 | NUMERIC_ITEM NUMERIC(5,3), 11 | SMALLMONEY_ITEM SMALLMONEY, 12 | MONEY_ITEM MONEY, 13 | REAL_ITEM REAL, 14 | FLOAT_ITEM FLOAT, 15 | CHAR_ITEM CHAR(4), 16 | VARCHAR_ITEM VARCHAR(8), 17 | TEXT_ITEM TEXT, 18 | NCHAR_ITEM NCHAR(4), 19 | NVARCHAR_ITEM NVARCHAR(8), 20 | NTEXT_ITEM NTEXT, 21 | DATE_ITEM DATE, 22 | DATETIME_ITEM DATETIME, 23 | DATETIME2_ITEM DATETIME2, 24 | DATETIME2_2_ITEM DATETIME2(2), 25 | SMALLDATETIME_ITEM SMALLDATETIME, 26 | TIME_ITEM TIME, 27 | TIME_2_ITEM TIME(2), 28 | PRIMARY KEY (ID) 29 | ); 30 | INSERT INTO TEST1(ID) VALUES('1000'); 31 | INSERT INTO TEST1 VALUES( 32 | '1001', 33 | 12, 34 | 1234, 35 | 123456, 36 | 12345678901234, 37 | 1, 38 | 1234567890.12, 39 | 12.345, 40 | 12.3456, 41 | 1234.5678, 42 | 1234567.8, 43 | 123456789012.34, 44 | 'a', 45 | 'A', 46 | 'abcdefg', 47 | N'あ', 48 | N'ア', 49 | N'あいうえお', 50 | '2018-01-02', 51 | '2018-01-03 12:34:56.789', 52 | '2018-01-04 12:34:56.1234567', 53 | '2018-01-05 12:34:56.89', 54 | '2018-01-06 12:34:56', 55 | '17:34:56.1234567', 56 | '18:34:56.12' 57 | ); 58 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/basic/test_config.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | order_by: 'ID' 3 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/basic/test_expected.csv: -------------------------------------------------------------------------------- 1 | 1000,,,,,,,,,,,,,,,,,,,,,,,, 2 | 1001,12,1234,123456,12345678901234,true,1.23456789012E9,12.345,12.3456,1234.5678,1234567.75,1.2345678901234E11,a ,A,abcdefg,あ ,ア,あいうえお,2018-01-02,2018-01-03 10:34:56.790000 +0000,2018-01-04 12:34:56.1234567,2018-01-05 12:34:56.89,2018-01-06 10:35:00.000000 +0000,17:34:56.1234567,18:34:56.12 3 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/basic/test_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/config_1.yml: -------------------------------------------------------------------------------- 1 | table: CHAR_LOAD 2 | incremental: true 3 | incremental_columns: [NAME] 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/config_2.yml: -------------------------------------------------------------------------------- 1 | table: CHAR_LOAD 2 | last_record: ['A4'] 3 | incremental: true 4 | incremental_columns: [NAME] 5 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_1.csv: -------------------------------------------------------------------------------- 1 | A1,first 2 | A2,first 3 | A3,first 4 | A4,first 5 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['A4'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_2.csv: -------------------------------------------------------------------------------- 1 | A5,more_load 2 | A9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['A9'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A0', 'more_skip'); 3 | INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A4', 'more_skip'); 4 | INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A9', 'more_load'); 5 | INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A5', 'more_load'); 6 | 7 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/setup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE CHAR_LOAD 2 | CREATE TABLE CHAR_LOAD ( 3 | NAME CHAR(2) NOT NULL, 4 | NOTE TEXT 5 | ); 6 | 7 | INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A3', 'first'); 8 | INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A4', 'first'); 9 | INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A2', 'first'); 10 | INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A1', 'first'); 11 | 12 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/config_1.yml: -------------------------------------------------------------------------------- 1 | table: DT_LOAD 2 | default_time_zone: +0300 # should be ignored 3 | incremental: true 4 | incremental_columns: [TIME] 5 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/config_2.yml: -------------------------------------------------------------------------------- 1 | table: DT_LOAD 2 | last_record: ['2016-11-02 04:00:05.3330030'] 3 | incremental: true 4 | incremental_columns: [TIME] 5 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_1.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 01:00:01.0000000,first 2 | 2016-11-02 02:00:02.0000000,first 3 | 2016-11-02 03:00:03.0000000,first 4 | 2016-11-02 04:00:04.0000000,first 5 | 2016-11-02 04:00:05.1110010,first 6 | 2016-11-02 04:00:05.2220020,first 7 | 2016-11-02 04:00:05.3330030,first 8 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02 04:00:05.3330030'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_2.csv: -------------------------------------------------------------------------------- 1 | 2016-11-02 04:00:05.3330040,more_load 2 | 2016-11-02 04:00:06.0000000,more_load 3 | 2016-11-02 04:00:06.0000000,more_load 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: ['2016-11-02 04:00:06.0000000'] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:00', 'more_skip'); 3 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.333000', 'more_skip'); 4 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.333003', 'more_skip'); 5 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.333004', 'more_load'); 6 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:06', 'more_load'); 7 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:06', 'more_load'); 8 | 9 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/setup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE DT_LOAD 2 | CREATE TABLE DT_LOAD ( 3 | TIME DATETIME2 NOT NULL, 4 | NOTE TEXT 5 | ); 6 | 7 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 01:00:01', 'first'); 8 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 02:00:02', 'first'); 9 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 03:00:03', 'first'); 10 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:04', 'first'); 11 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.111001', 'first'); 12 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.222002', 'first'); 13 | INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.333003', 'first'); 14 | 15 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/config_1.yml: -------------------------------------------------------------------------------- 1 | table: INT_LOAD 2 | incremental: true 3 | incremental_columns: [NUM] 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/config_2.yml: -------------------------------------------------------------------------------- 1 | table: INT_LOAD 2 | last_record: [4] 3 | incremental: true 4 | incremental_columns: [NUM] 5 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,first 2 | 2,first 3 | 3,first 4 | 4,first 5 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,more_load 2 | 9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO INT_LOAD (NUM, NOTE) VALUES (0, 'more_skip'); 3 | INSERT INTO INT_LOAD (NUM, NOTE) VALUES (4, 'more_skip'); 4 | INSERT INTO INT_LOAD (NUM, NOTE) VALUES (9, 'more_load'); 5 | INSERT INTO INT_LOAD (NUM, NOTE) VALUES (5, 'more_load'); 6 | 7 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/setup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE INT_LOAD 2 | CREATE TABLE INT_LOAD ( 3 | NUM INT NOT NULL, 4 | NOTE TEXT 5 | ); 6 | 7 | INSERT INTO INT_LOAD (NUM, NOTE) VALUES (3, 'first'); 8 | INSERT INTO INT_LOAD (NUM, NOTE) VALUES (4, 'first'); 9 | INSERT INTO INT_LOAD (NUM, NOTE) VALUES (2, 'first'); 10 | INSERT INTO INT_LOAD (NUM, NOTE) VALUES (1, 'first'); 11 | 12 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0] 2 | incremental: true 3 | incremental_columns: [NUM] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | NUM,NOTE 8 | FROM 9 | QUERY_LOAD 10 | WHERE 11 | NUM IS NOT NULL 12 | AND NUM > :NUM 13 | ORDER BY 14 | NUM ASC 15 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4] 2 | incremental: true 3 | incremental_columns: [NUM] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | NUM,NOTE 8 | FROM 9 | QUERY_LOAD 10 | WHERE 11 | NUM IS NOT NULL 12 | AND NUM > :NUM 13 | ORDER BY 14 | NUM ASC 15 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,first 2 | 2,first 3 | 3,first 4 | 4,first 5 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,more_load 2 | 9,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/insert_more.sql: -------------------------------------------------------------------------------- 1 | 2 | INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (0, 100, 'more_skip'); 3 | INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (4, 104, 'more_skip'); 4 | INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (9, 109, 'more_load'); 5 | INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (5, 105, 'more_load'); 6 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_config_1.yml: -------------------------------------------------------------------------------- 1 | last_record: [0,0] 2 | incremental: true 3 | incremental_columns: [NUM,NUM2] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | * 8 | FROM 9 | QUERY_LOAD 10 | WHERE 11 | NUM IS NOT NULL 12 | AND NUM > :NUM 13 | OR (NUM = :NUM AND NUM2 > :NUM2) 14 | ORDER BY 15 | NUM ASC, 16 | NUM2 ASC 17 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_config_2.yml: -------------------------------------------------------------------------------- 1 | last_record: [4,104] 2 | incremental: true 3 | incremental_columns: [NUM,NUM2] 4 | use_raw_query_with_incremental: true 5 | query: | 6 | SELECT 7 | * 8 | FROM 9 | QUERY_LOAD 10 | WHERE 11 | NUM IS NOT NULL 12 | AND NUM > :NUM 13 | AND NUM2 IS NOT NULL 14 | OR (NUM = :NUM AND NUM2 > :NUM2) 15 | ORDER BY 16 | NUM ASC, 17 | NUM2 ASC 18 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_1.csv: -------------------------------------------------------------------------------- 1 | 1,101,first 2 | 2,102,first 3 | 3,103,first 4 | 4,104,first 5 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_1.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [4,104] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_2.csv: -------------------------------------------------------------------------------- 1 | 5,105,more_load 2 | 9,109,more_load 3 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_2.diff: -------------------------------------------------------------------------------- 1 | in: 2 | last_record: [9,109] 3 | out: {} 4 | -------------------------------------------------------------------------------- /embulk-input-sqlserver/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/setup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE QUERY_LOAD 2 | CREATE TABLE QUERY_LOAD ( 3 | NUM INT NOT NULL, 4 | NUM2 INT NOT NULL, 5 | NOTE TEXT 6 | ); 7 | 8 | INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (3, 103, 'first'); 9 | INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (4, 104, 'first'); 10 | INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (2, 102, 'first'); 11 | INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (1, 101, 'first'); 12 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/embulk/embulk-input-jdbc/068d6b22fca6447e23de71d6df9e95e5e1dc3f72/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.13-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | @rem SPDX-License-Identifier: Apache-2.0 17 | @rem 18 | 19 | @if "%DEBUG%"=="" @echo off 20 | @rem ########################################################################## 21 | @rem 22 | @rem Gradle startup script for Windows 23 | @rem 24 | @rem ########################################################################## 25 | 26 | @rem Set local scope for the variables with windows NT shell 27 | if "%OS%"=="Windows_NT" setlocal 28 | 29 | set DIRNAME=%~dp0 30 | if "%DIRNAME%"=="" set DIRNAME=. 31 | @rem This is normally unused 32 | set APP_BASE_NAME=%~n0 33 | set APP_HOME=%DIRNAME% 34 | 35 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 36 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 37 | 38 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 39 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 40 | 41 | @rem Find java.exe 42 | if defined JAVA_HOME goto findJavaFromJavaHome 43 | 44 | set JAVA_EXE=java.exe 45 | %JAVA_EXE% -version >NUL 2>&1 46 | if %ERRORLEVEL% equ 0 goto execute 47 | 48 | echo. 1>&2 49 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 50 | echo. 1>&2 51 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 52 | echo location of your Java installation. 1>&2 53 | 54 | goto fail 55 | 56 | :findJavaFromJavaHome 57 | set JAVA_HOME=%JAVA_HOME:"=% 58 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 59 | 60 | if exist "%JAVA_EXE%" goto execute 61 | 62 | echo. 1>&2 63 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 64 | echo. 1>&2 65 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 66 | echo location of your Java installation. 1>&2 67 | 68 | goto fail 69 | 70 | :execute 71 | @rem Setup the command line 72 | 73 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 74 | 75 | 76 | @rem Execute Gradle 77 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 78 | 79 | :end 80 | @rem End local scope for the variables with windows NT shell 81 | if %ERRORLEVEL% equ 0 goto mainEnd 82 | 83 | :fail 84 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 85 | rem the _cmd.exe /c_ return code! 86 | set EXIT_CODE=%ERRORLEVEL% 87 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 88 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 89 | exit /b %EXIT_CODE% 90 | 91 | :mainEnd 92 | if "%OS%"=="Windows_NT" endlocal 93 | 94 | :omega 95 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'embulk-input-jdbc-root' 2 | include 'embulk-input-jdbc' 3 | include 'embulk-input-mysql' 4 | include 'embulk-input-postgresql' 5 | include 'embulk-input-redshift' 6 | include 'embulk-input-sqlserver' 7 | -------------------------------------------------------------------------------- /test-scripts/mysql-input/data/test_expected.csv: -------------------------------------------------------------------------------- 1 | ID,NUM,STR,VARSTR,DT,DTTM0,DTTM3 2 | 1,123.40,test1,TEST1,2015-04-24,2015-04-24 10:02:03,2015-04-24 10:02:03.123 3 | 2,1234567890.12,test9999,TEST9999,2015-12-31,2016-01-01 08:59:59,2016-01-01 08:59:59.999 4 | 3,,,,,, 5 | -------------------------------------------------------------------------------- /test-scripts/mysql-input/test.bat: -------------------------------------------------------------------------------- 1 | mysql -uTEST_USER -pXXXXXXXX -DTESTDB < test.sql 2 | 3 | del data\test000.00.csv 4 | CALL embulk run test.yml 5 | 6 | echo "diff data/test_expected.csv data/test000.00.csv" 7 | diff data/test_expected.csv data/test000.00.csv 8 | 9 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-input-mysql FAILED!") 10 | -------------------------------------------------------------------------------- /test-scripts/mysql-input/test.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS EMBULK_INPUT; 2 | 3 | CREATE TABLE EMBULK_INPUT ( 4 | ID INT, 5 | NUM DECIMAL(12,2), 6 | STR CHAR(8), 7 | VARSTR VARCHAR(8), 8 | DT DATE, 9 | DTTM0 DATETIME, 10 | DTTM3 DATETIME(3), 11 | PRIMARY KEY(ID) 12 | ); 13 | 14 | 15 | INSERT INTO EMBULK_INPUT VALUES( 16 | 1, 17 | 123.4, 18 | 'test1', 19 | 'TEST1', 20 | '2015-04-24', 21 | '2015-04-24 01:02:03', 22 | '2015-04-24 01:02:03.123' 23 | ); 24 | 25 | INSERT INTO EMBULK_INPUT VALUES( 26 | 2, 27 | 1234567890.12, 28 | 'test9999', 29 | 'TEST9999', 30 | '2015-12-31', 31 | '2015-12-31 23:59:59', 32 | '2015-12-31 23:59:59.999' 33 | ); 34 | 35 | INSERT INTO EMBULK_INPUT VALUES( 36 | 3, 37 | NULL, 38 | NULL, 39 | NULL, 40 | NULL, 41 | NULL, 42 | NULL 43 | ); 44 | -------------------------------------------------------------------------------- /test-scripts/mysql-input/test.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: mysql 3 | host: localhost 4 | database: TESTDB 5 | user: TEST_USER 6 | password: XXXXXXXX 7 | table: EMBULK_INPUT 8 | select: "*" 9 | column_options: 10 | NUM: {type: string} 11 | DT: {type: string, timestamp_format: '%Y-%m-%d', timezone: "+0900"} 12 | DTTM0: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 13 | DTTM3: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S.%3N', timezone: "+0900"} 14 | out: 15 | type: file 16 | path_prefix: data/test 17 | file_ext: csv 18 | formatter: 19 | type: csv 20 | -------------------------------------------------------------------------------- /test-scripts/postgresql-input/data/test_expected.csv: -------------------------------------------------------------------------------- 1 | id,num,str,varstr,dt,dttm0,dttm3 2 | 1,123.40,test1 ,TEST1,2015-04-24,2015-04-24 01:02:03,2015-04-24 01:02:03.123 3 | 2,1234567890.12,test9999,TEST9999,2015-12-31,2015-12-31 23:59:59,2015-12-31 23:59:59.999 4 | 3,,,,,, 5 | -------------------------------------------------------------------------------- /test-scripts/postgresql-input/test.bat: -------------------------------------------------------------------------------- 1 | SETLOCAL 2 | SET PGPASSWORD=XXXXXXXX 3 | "C:\Program Files\PostgreSQL\9.4\bin\psql.exe" -d testdb -U test_user -w -f test.sql 4 | ENDLOCAL 5 | 6 | del data\test000.00.csv 7 | CALL embulk run test.yml 8 | 9 | echo "diff data/test_expected.csv data/test000.00.csv" 10 | diff data/test_expected.csv data/test000.00.csv 11 | 12 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-input-postgresql FAILED!") 13 | -------------------------------------------------------------------------------- /test-scripts/postgresql-input/test.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS embulk_input; 2 | 3 | CREATE TABLE embulk_input ( 4 | id int, 5 | num decimal(12,2), 6 | str char(8), 7 | varstr varchar(8), 8 | dt date, 9 | dttm0 timestamp, 10 | dttm3 timestamp(3), 11 | primary key(ID) 12 | ); 13 | 14 | 15 | INSERT INTO embulk_input VALUES( 16 | 1, 17 | 123.4, 18 | 'test1', 19 | 'TEST1', 20 | '2015-04-24', 21 | '2015-04-24 01:02:03', 22 | '2015-04-24 01:02:03.123' 23 | ); 24 | 25 | INSERT INTO embulk_input VALUES( 26 | 2, 27 | 1234567890.12, 28 | 'test9999', 29 | 'TEST9999', 30 | '2015-12-31', 31 | '2015-12-31 23:59:59', 32 | '2015-12-31 23:59:59.999' 33 | ); 34 | 35 | INSERT INTO embulk_input VALUES( 36 | 3, 37 | NULL, 38 | NULL, 39 | NULL, 40 | NULL, 41 | NULL, 42 | NULL 43 | ); 44 | -------------------------------------------------------------------------------- /test-scripts/postgresql-input/test.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: postgresql 3 | host: localhost 4 | database: testdb 5 | user: test_user 6 | password: XXXXXXXX 7 | table: embulk_input 8 | select: "*" 9 | column_options: 10 | num: {type: string} 11 | dt: {type: string, timestamp_format: '%Y-%m-%d', timezone: "+0900"} 12 | dttm0: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 13 | dttm3: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S.%3N', timezone: "+0900"} 14 | out: 15 | type: file 16 | path_prefix: data/test 17 | file_ext: csv 18 | formatter: 19 | type: csv 20 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-input/data/test_expected.csv: -------------------------------------------------------------------------------- 1 | ID,NUM,STR,VARSTR,DT,DTTM,DTTM2,SDTTM 2 | 1,123.40,test1 ,TEST1,2015-04-24,2015-04-24 01:02:03.123,2015-04-24 01:02:03.1234567,2015-04-24 01:02:00 3 | 2,1234567890.12,test9999,TEST9999,2015-12-31,2015-12-31 23:59:59.993,2015-12-31 23:59:59.9999999,2016-01-01 00:00:00 4 | 3,,,,,,, 5 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-input/test-jtds.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test.sql 2 | 3 | del data\test000.00.csv 4 | CALL embulk run test-jtds.yml 5 | 6 | echo "diff data/test_expected.csv data/test000.00.csv" 7 | diff data/test_expected.csv data/test000.00.csv 8 | 9 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-input-sqlserver FAILED!") 10 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-input/test-jtds.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: sqlserver 3 | application_name: embulk_test(jtds) 4 | host: localhost 5 | port: 1433 6 | database: TESTDB 7 | user: TEST_USER 8 | password: XXXXXXXX 9 | table: EMBULK_INPUT 10 | select: "*" 11 | column_options: 12 | ID: {type: string} 13 | NUM: {type: string} 14 | DT: {type: string, timestamp_format: '%Y-%m-%d', timezone: "+0900"} 15 | DTTM: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S.%3N', timezone: "+0900"} 16 | DTTM2: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S.%7N', timezone: "+0900"} 17 | SDTTM: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 18 | out: 19 | type: file 20 | path_prefix: data/test 21 | file_ext: csv 22 | formatter: 23 | type: csv 24 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-input/test.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test.sql 2 | 3 | del data\test000.00.csv 4 | CALL embulk run test.yml 5 | 6 | echo "diff data/test_expected.csv data/test000.00.csv" 7 | diff data/test_expected.csv data/test000.00.csv 8 | 9 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-input-sqlserver FAILED!") 10 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-input/test.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE EMBULK_INPUT; 2 | 3 | CREATE TABLE EMBULK_INPUT ( 4 | ID DECIMAL(4, 0), 5 | NUM DECIMAL(12,2), 6 | STR CHAR(8), 7 | VARSTR VARCHAR(8), 8 | DT DATE, 9 | DTTM DATETIME, 10 | DTTM2 DATETIME2, 11 | SDTTM SMALLDATETIME, 12 | PRIMARY KEY(ID) 13 | ); 14 | 15 | 16 | INSERT INTO EMBULK_INPUT VALUES( 17 | 1, 18 | 123.4, 19 | 'test1', 20 | 'TEST1', 21 | '2015-04-24', 22 | '2015-04-24 01:02:03.123', 23 | '2015-04-24 01:02:03.1234567', 24 | '2015-04-24 01:02:03' 25 | ); 26 | 27 | INSERT INTO EMBULK_INPUT VALUES( 28 | 2, 29 | 1234567890.12, 30 | 'test9999', 31 | 'TEST9999', 32 | '2015-12-31', 33 | '2015-12-31 23:59:59.993', 34 | '2015-12-31 23:59:59.9999999', 35 | '2015-12-31 23:59:59' 36 | ); 37 | 38 | INSERT INTO EMBULK_INPUT VALUES( 39 | 3, 40 | NULL, 41 | NULL, 42 | NULL, 43 | NULL, 44 | NULL, 45 | NULL, 46 | NULL 47 | ); 48 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-input/test.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: sqlserver 3 | driver_path: driver\sqljdbc41.jar 4 | application_name: embulk_test 5 | host: localhost 6 | port: 1433 7 | database: TESTDB 8 | user: TEST_USER 9 | password: XXXXXXXX 10 | table: EMBULK_INPUT 11 | select: "*" 12 | column_options: 13 | ID: {type: string} 14 | NUM: {type: string} 15 | DT: {type: string, timestamp_format: '%Y-%m-%d', timezone: "+0900"} 16 | DTTM: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S.%3N', timezone: "+0900"} 17 | DTTM2: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S.%7N', timezone: "+0900"} 18 | SDTTM: {type: string, timestamp_format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 19 | out: 20 | type: file 21 | path_prefix: data/test 22 | file_ext: csv 23 | formatter: 24 | type: csv 25 | -------------------------------------------------------------------------------- /test-scripts/test-input.bat: -------------------------------------------------------------------------------- 1 | set LOG=%~dp0test-input.log 2 | echo log file = %LOG% 3 | del %LOG% 4 | 5 | cd mysql-input 6 | echo "mysql-input/test.bat" 7 | call test.bat >> %LOG% 8 | cd .. 9 | 10 | cd postgresql-input 11 | echo "postgresql-input/test.bat" 12 | call test.bat >> %LOG% 13 | cd .. 14 | 15 | cd sqlserver-input 16 | echo "sqlserver-input/test.bat" 17 | call test.bat >> %LOG% 18 | echo "sqlserver-input/test-jtds.bat" 19 | call test-jtds.bat >> %LOG% 20 | cd .. 21 | 22 | grep "FAILED" %LOG% 23 | 24 | IF "%ERRORLEVEL%" == "0" (ECHO "FAILED!") ELSE (ECHO "OK!") 25 | --------------------------------------------------------------------------------