├── .github ├── CODEOWNERS └── workflows │ ├── check.yml │ └── main.yml ├── .gitignore ├── COPYING ├── ChangeLog ├── README.md ├── build.gradle ├── ci ├── mysql.yml ├── postgresql.yml └── redshift.yml ├── config └── checkstyle │ ├── checkstyle.xml │ └── default.xml ├── embulk-output-jdbc ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── output │ │ ├── JdbcOutputPlugin.java │ │ └── jdbc │ │ ├── AbstractJdbcOutputConnector.java │ │ ├── AbstractJdbcOutputPlugin.java │ │ ├── BatchInsert.java │ │ ├── JdbcColumn.java │ │ ├── JdbcColumnOption.java │ │ ├── JdbcOutputConnection.java │ │ ├── JdbcOutputConnector.java │ │ ├── JdbcSchema.java │ │ ├── JdbcUtils.java │ │ ├── MemoryRecord.java │ │ ├── MergeConfig.java │ │ ├── PageReaderRecord.java │ │ ├── Record.java │ │ ├── Ssl.java │ │ ├── StandardBatchInsert.java │ │ ├── TableIdentifier.java │ │ ├── TimestampFormat.java │ │ ├── ToString.java │ │ ├── ToStringMap.java │ │ ├── TransactionIsolation.java │ │ └── setter │ │ ├── BigDecimalColumnSetter.java │ │ ├── BooleanColumnSetter.java │ │ ├── ByteColumnSetter.java │ │ ├── ColumnSetter.java │ │ ├── ColumnSetterFactory.java │ │ ├── ColumnSetterVisitor.java │ │ ├── DefaultValueSetter.java │ │ ├── DoubleColumnSetter.java │ │ ├── FloatColumnSetter.java │ │ ├── IntColumnSetter.java │ │ ├── JsonColumnSetter.java │ │ ├── LongColumnSetter.java │ │ ├── NStringColumnSetter.java │ │ ├── NullColumnSetter.java │ │ ├── NullDefaultValueSetter.java │ │ ├── PassThroughColumnSetter.java │ │ ├── ShortColumnSetter.java │ │ ├── SkipColumnSetter.java │ │ ├── SqlDateColumnSetter.java │ │ ├── SqlTimeColumnSetter.java │ │ ├── SqlTimestampColumnSetter.java │ │ └── StringColumnSetter.java │ └── test │ └── java │ └── org │ └── embulk │ └── output │ └── jdbc │ ├── AbstractJdbcOutputPluginTest.java │ ├── TestJdbcOutputPlugin.java │ ├── TimestampFormatTest.java │ └── tester │ └── EmbulkPluginTester.java ├── embulk-output-mysql ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── output │ │ ├── MySQLOutputPlugin.java │ │ ├── MySQLTimeZoneComparison.java │ │ └── mysql │ │ ├── MySQLBatchInsert.java │ │ ├── MySQLOutputConnection.java │ │ └── MySQLOutputConnector.java │ └── test │ ├── java │ └── org │ │ └── embulk │ │ └── output │ │ └── mysql │ │ ├── AfterLoadTest.java │ │ ├── BasicTest.java │ │ ├── BeforeLoadTest.java │ │ ├── CreateTableTest.java │ │ ├── MySQLTests.java │ │ └── RetryTest.java │ └── resources │ └── org │ └── embulk │ └── output │ └── mysql │ └── test │ └── expect │ ├── after_load │ ├── setup.sql │ ├── test1.csv │ ├── test_expected.diff │ ├── test_insert_after_load.yml │ ├── test_insert_after_load_expected.csv │ ├── test_insert_direct_after_load.yml │ ├── test_merge_after_load.yml │ ├── test_merge_after_load_expected.csv │ ├── test_merge_direct_after_load.yml │ ├── test_replace_after_load.yml │ ├── test_replace_after_load_expected.csv │ ├── test_truncate_insert_after_load.yml │ └── test_truncate_insert_after_load_expected.csv │ ├── basic │ ├── setup.sql │ ├── test1.csv │ ├── test_invalid_time_zone.yml │ ├── test_merge.csv │ ├── test_merge.yml │ ├── test_merge_expected.csv │ ├── test_replace.yml │ └── test_replace_expected.csv │ ├── before_load │ ├── setup.sql │ ├── test1.csv │ ├── test_expected.diff │ ├── test_insert_before_load.yml │ ├── test_insert_before_load_expected.csv │ ├── test_insert_direct_before_load.yml │ ├── test_merge_before_load.yml │ ├── test_merge_before_load_expected.csv │ ├── test_merge_direct_before_load.yml │ ├── test_truncate_insert_before_load.yml │ └── test_truncate_insert_before_load_expected.csv │ ├── create_table │ ├── setup.sql │ ├── test1.csv │ ├── test_table_constraint.yml │ ├── test_table_constraint_expected.csv │ ├── test_table_option.yml │ └── test_table_option_expected.csv │ └── retry │ ├── setup.sql │ ├── test1.csv │ ├── test1.yml │ ├── test1_expected.csv │ ├── test1_flushed_multiple_times.csv │ ├── test1_flushed_multiple_times.yml │ └── test1_flushed_multiple_times_expected.csv ├── embulk-output-postgresql ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── output │ │ ├── PostgreSQLOutputPlugin.java │ │ └── postgresql │ │ ├── AbstractPostgreSQLCopyBatchInsert.java │ │ ├── PostgreSQLCopyBatchInsert.java │ │ ├── PostgreSQLOutputConnection.java │ │ ├── PostgreSQLOutputConnector.java │ │ └── setter │ │ └── PostgreSQLColumnSetterFactory.java │ └── test │ ├── java │ └── org │ │ └── embulk │ │ └── output │ │ └── postgresql │ │ ├── AfterLoadTest.java │ │ ├── BasicTest.java │ │ ├── BeforeLoadTest.java │ │ └── PostgreSQLTests.java │ └── resources │ └── org │ └── embulk │ └── output │ └── postgresql │ └── test │ └── expect │ ├── after_load │ ├── setup.sql │ ├── test1.csv │ ├── test_expected.diff │ ├── test_insert_after_load.yml │ ├── test_insert_after_load_expected.csv │ ├── test_insert_direct_after_load.yml │ ├── test_merge_after_load.yml │ ├── test_merge_after_load_expected.csv │ ├── test_merge_direct_after_load.yml │ ├── test_replace_after_load.yml │ ├── test_replace_after_load_expected.csv │ ├── test_truncate_insert_after_load.yml │ └── test_truncate_insert_after_load_expected.csv │ ├── basic │ ├── setup.sql │ ├── test_json.csv │ ├── test_json.yml │ ├── test_json_expected.csv │ ├── test_merge.csv │ ├── test_merge.yml │ ├── test_merge_expected.csv │ ├── test_merge_keys.csv │ ├── test_merge_keys.yml │ ├── test_merge_keys_expected.csv │ ├── test_merge_rule.yml │ ├── test_merge_rule_expected.csv │ ├── test_number.csv │ ├── test_number.yml │ ├── test_number_expected.csv │ ├── test_replace.yml │ ├── test_replace_expected.csv │ ├── test_string.csv │ ├── test_string.yml │ ├── test_string_expected.csv │ ├── test_timestamp.csv │ ├── test_timestamp.yml │ └── test_timestamp_expected.csv │ └── before_load │ ├── setup.sql │ ├── test1.csv │ ├── test_expected.diff │ ├── test_insert_before_load.yml │ ├── test_insert_before_load_expected.csv │ ├── test_insert_direct_before_load.yml │ ├── test_merge_before_load.yml │ ├── test_merge_before_load_expected.csv │ ├── test_merge_direct_before_load.yml │ ├── test_truncate_insert_before_load.yml │ └── test_truncate_insert_before_load_expected.csv ├── embulk-output-redshift ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ └── main │ └── java │ └── org │ └── embulk │ └── output │ ├── RedshiftOutputPlugin.java │ └── redshift │ ├── RedshiftCopyBatchInsert.java │ ├── RedshiftOutputConnection.java │ └── RedshiftOutputConnector.java ├── embulk-output-sqlserver ├── README.md ├── build.gradle ├── gradle.lockfile └── src │ ├── main │ └── java │ │ └── org │ │ └── embulk │ │ └── output │ │ ├── SQLServerOutputPlugin.java │ │ └── sqlserver │ │ ├── InsertMethod.java │ │ ├── NativeBatchInsert.java │ │ ├── SQLServerOutputConnection.java │ │ ├── SQLServerOutputConnector.java │ │ ├── SmallDateTimeFormat.java │ │ ├── nativeclient │ │ ├── NativeClient.java │ │ ├── NativeClientWrapper.java │ │ └── ODBC.java │ │ └── setter │ │ ├── SQLServerByteColumnSetter.java │ │ ├── SQLServerColumnSetterFactory.java │ │ └── SQLServerSqlTimeColumnSetter.java │ └── test │ ├── java │ └── org │ │ └── embulk │ │ └── output │ │ └── sqlserver │ │ ├── BasicTest.java │ │ ├── NativeTest.java │ │ └── SQLServerTests.java │ └── resources │ └── org │ └── embulk │ └── output │ └── sqlserver │ └── test │ └── expect │ ├── basic │ ├── setup.sql │ ├── test1.csv │ ├── test_expected.diff │ ├── test_insert.yml │ ├── test_insert_create_expected.csv │ ├── test_insert_direct.yml │ ├── test_insert_expected.csv │ ├── test_jtds.yml │ ├── test_max.yml │ ├── test_merge.csv │ ├── test_merge.yml │ ├── test_merge_expected.csv │ ├── test_merge_keys.yml │ ├── test_merge_rule.yml │ ├── test_merge_rule_expected.csv │ ├── test_native_insert_direct.yml │ ├── test_replace.yml │ ├── test_replace_longname.yml │ ├── test_string_timestamp.csv │ ├── test_string_timestamp.yml │ ├── test_string_timestamp_expected.csv │ ├── test_truncate_insert.yml │ └── test_truncate_insert_expected.csv │ └── native │ ├── setup.sql │ ├── test_bigint_null.yml │ ├── test_bit_null.csv │ ├── test_bit_null.yml │ ├── test_bit_null_expected.csv │ ├── test_char_null.csv │ ├── test_char_null.yml │ ├── test_char_null_expected.csv │ ├── test_date_null.yml │ ├── test_date_null_expected.csv │ ├── test_datetime2_null.yml │ ├── test_datetime2_null_expected.csv │ ├── test_datetime_null.yml │ ├── test_datetime_null_expected.csv │ ├── test_decimal_null.csv │ ├── test_decimal_null.yml │ ├── test_decimal_null_expected.csv │ ├── test_float_null.csv │ ├── test_float_null.yml │ ├── test_float_null_expected.csv │ ├── test_huge.csv │ ├── test_huge.yml │ ├── test_int_null.yml │ ├── test_integer_null.csv │ ├── test_integer_null_expected.csv │ ├── test_money_null.yml │ ├── test_nchar_null.yml │ ├── test_ntext_null.yml │ ├── test_numeric_null.yml │ ├── test_nvarchar_null.yml │ ├── test_real_null.yml │ ├── test_smalldatetime_null.yml │ ├── test_smalldatetime_null_expected.csv │ ├── test_smallint_null.yml │ ├── test_smallmoney_null.yml │ ├── test_text_null.yml │ ├── test_time_null.yml │ ├── test_time_null_expected.csv │ ├── test_timestamp_null.csv │ ├── test_tinyint_null.yml │ ├── test_varchar_null.csv │ ├── test_varchar_null.yml │ └── test_varchar_null_expected.csv ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── test-scripts ├── mysql-output ├── data │ ├── test.csv │ ├── test_expected.txt │ └── test_replace_expected.txt ├── test.bat ├── test.sql ├── test.yml ├── test_temp_database.bat ├── test_temp_database.yml ├── test_temp_database_replace.bat └── test_temp_database_replace.yml ├── postgresql-output ├── data │ ├── test-replace_expected.txt │ ├── test.csv │ ├── test2.csv │ └── test_expected.txt ├── test-temp-schema-replace.bat ├── test-temp-schema-replace.yml ├── test-temp-schema.bat ├── test-temp-schema.yml ├── test.bat ├── test.sql └── test.yml ├── sqlserver-output ├── data │ ├── test-replace_expected.txt │ ├── test.csv │ └── test_expected.txt ├── test-jtds.bat ├── test-jtds.yml ├── test-native.bat ├── test-native.yml ├── test-replace-schema.bat ├── test-replace-schema.yml ├── test-replace-temp-schema.bat ├── test-replace-temp-schema.yml ├── test-schema.bat ├── test-schema.sql ├── test-schema.yml ├── test-temp-schema.bat ├── test-temp-schema.yml ├── test.bat ├── test.sql └── test.yml └── test-output.bat /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @embulk/jdbc-maintainers 2 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Sync master to main 2 | on: 3 | push: 4 | branches: 5 | - "master" 6 | jobs: 7 | diff: 8 | runs-on: ubuntu-latest 9 | outputs: 10 | hash-master: ${{ steps.hash-master.outputs.hash-master }} 11 | hash-main: ${{ steps.hash-main.outputs.hash-main }} 12 | steps: 13 | - uses: actions/checkout@v3 14 | with: 15 | fetch-depth: 0 16 | - id: hash-master 17 | name: Hash the master branch 18 | run: | 19 | hash_master=$( git rev-parse origin/master ) 20 | echo "$hash_master" 21 | echo "hash-master=$hash_master" >> $GITHUB_OUTPUT 22 | - id: hash-main 23 | name: Hash the main branch 24 | run: | 25 | hash_main=$( git rev-parse origin/main ) 26 | echo "$hash_main" 27 | echo "hash-main=$hash_main" >> $GITHUB_OUTPUT 28 | sync: 29 | needs: diff 30 | if: needs.diff.outputs.hash-master != needs.diff.outputs.hash-main 31 | runs-on: ubuntu-latest 32 | steps: 33 | - uses: actions/checkout@v3 34 | with: 35 | fetch-depth: 0 36 | - name: Checkout master 37 | run: git checkout master 38 | - name: Sync master to main 39 | run: git push origin master:main 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | /tmp/ 3 | .gradle/ 4 | build/ 5 | .idea 6 | *.iml 7 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | Copyright (C) 2015 Sadayuki Furuhashi 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JDBC output plugins for Embulk 2 | 3 | JDBC output plugins for Embulk loads records to databases using JDBC drivers. 4 | 5 | **[WARNING!]** The next version of embulk-output-jdbc will require Embulk v0.11 and Java 8. It may work with Java 11+, but not fully tested. 6 | 7 | ## MySQL 8 | 9 | See [embulk-output-mysql](embulk-output-mysql/). 10 | 11 | ## PostgreSQL 12 | 13 | See [embulk-output-postgresql](embulk-output-postgresql/). 14 | 15 | ## Redshift 16 | 17 | See [embulk-output-redshift](embulk-output-redshift/). 18 | 19 | ## SQL Server 20 | 21 | See [embulk-output-sqlserver](embulk-output-sqlserver/). 22 | 23 | ## Others (generic JDBC) 24 | 25 | See [embulk-output-jdbc](embulk-output-jdbc/). 26 | -------------------------------------------------------------------------------- /ci/mysql.yml: -------------------------------------------------------------------------------- 1 | type: mysql 2 | host: 127.0.0.1 3 | port: 3306 4 | database: ci_test 5 | user: root 6 | password: root 7 | -------------------------------------------------------------------------------- /ci/postgresql.yml: -------------------------------------------------------------------------------- 1 | type: postgresql 2 | host: 127.0.0.1 3 | port: 5432 4 | database: ci_test 5 | user: postgres 6 | password: postgres 7 | -------------------------------------------------------------------------------- /ci/redshift.yml: -------------------------------------------------------------------------------- 1 | type: redshift 2 | host: 127.0.0.1 3 | port: 5439 4 | database: ci_test 5 | user: postgres 6 | password: postgres 7 | fetch_rows: 1000 8 | -------------------------------------------------------------------------------- /embulk-output-jdbc/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | } 3 | 4 | embulkPlugin { 5 | mainClass = "org.embulk.output.JdbcOutputPlugin" 6 | category = "output" 7 | type = "jdbc" 8 | } 9 | 10 | publishing { 11 | publications { 12 | maven(MavenPublication) { 13 | pom { // https://central.sonatype.org/pages/requirements.html 14 | developers { 15 | developer { 16 | name = "Sadayuki Furuhashi" 17 | email = "frsyuki@gmail.com" 18 | } 19 | developer { 20 | name = "Hitoshi Tanaka" 21 | email = "thitoshi@cac.co.jp" 22 | } 23 | developer { 24 | name = "takumakanari" 25 | email = "chemtrails.t@gmail.com" 26 | } 27 | developer { 28 | name = "Tomohiro Hashidate" 29 | email = "kakyoin.hierophant@gmail.com" 30 | } 31 | developer { 32 | name = "Hiroyuki Sato" 33 | email = "hiroysato@gmail.com" 34 | } 35 | developer { 36 | name = "Dai MIKURUBE" 37 | email = "dmikurube@treasure-data.com" 38 | } 39 | } 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /embulk-output-jdbc/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 9 | org.embulk:embulk-spi:0.10.49=compileClasspath 10 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 11 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 12 | org.embulk:embulk-util-retryhelper:0.8.2=compileClasspath,runtimeClasspath 13 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 14 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 15 | org.msgpack:msgpack-core:0.8.24=compileClasspath 16 | org.slf4j:slf4j-api:2.0.7=compileClasspath 17 | empty= 18 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputConnector.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.sql.SQLException; 4 | import java.util.Optional; 5 | 6 | public abstract class AbstractJdbcOutputConnector implements JdbcOutputConnector 7 | { 8 | private final Optional transactionIsolation; 9 | 10 | public AbstractJdbcOutputConnector(Optional transactionIsolation) 11 | { 12 | this.transactionIsolation = transactionIsolation; 13 | } 14 | 15 | public JdbcOutputConnection connect(boolean autoCommit) throws SQLException 16 | { 17 | JdbcOutputConnection connection = connect(); 18 | connection.initialize(autoCommit, transactionIsolation); 19 | return connection; 20 | } 21 | 22 | protected abstract JdbcOutputConnection connect() throws SQLException; 23 | } 24 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/BatchInsert.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.math.BigDecimal; 4 | import java.util.Calendar; 5 | import java.io.IOException; 6 | import java.sql.SQLException; 7 | import java.time.Instant; 8 | 9 | public interface BatchInsert 10 | { 11 | public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException; 12 | 13 | public int getBatchWeight(); 14 | 15 | public void add() throws IOException, SQLException; 16 | 17 | public void close() throws IOException, SQLException; 18 | 19 | public void flush() throws IOException, SQLException; 20 | 21 | // should be implemented for retry 22 | public int[] getLastUpdateCounts(); 23 | 24 | public void finish() throws IOException, SQLException; 25 | 26 | public void setNull(int sqlType) throws IOException, SQLException; 27 | 28 | public void setBoolean(boolean v) throws IOException, SQLException; 29 | 30 | public void setByte(byte v) throws IOException, SQLException; 31 | 32 | public void setShort(short v) throws IOException, SQLException; 33 | 34 | public void setInt(int v) throws IOException, SQLException; 35 | 36 | public void setLong(long v) throws IOException, SQLException; 37 | 38 | public void setFloat(float v) throws IOException, SQLException; 39 | 40 | public void setDouble(double v) throws IOException, SQLException; 41 | 42 | public void setBigDecimal(BigDecimal v) throws IOException, SQLException; 43 | 44 | public void setString(String v) throws IOException, SQLException; 45 | 46 | public void setNString(String v) throws IOException, SQLException; 47 | 48 | public void setBytes(byte[] v) throws IOException, SQLException; 49 | 50 | public void setSqlDate(Instant v, Calendar cal) throws IOException, SQLException; 51 | 52 | public void setSqlTime(Instant v, Calendar cal) throws IOException, SQLException; 53 | 54 | public void setSqlTimestamp(Instant v, Calendar cal) throws IOException, SQLException; 55 | } 56 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/JdbcColumnOption.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.time.ZoneId; 4 | import java.util.Optional; 5 | import org.embulk.util.config.Config; 6 | import org.embulk.util.config.ConfigDefault; 7 | import org.embulk.util.config.Task; 8 | 9 | public interface JdbcColumnOption 10 | extends Task 11 | { 12 | @Config("type") 13 | @ConfigDefault("null") 14 | public Optional getType(); 15 | 16 | @Config("value_type") 17 | @ConfigDefault("\"coerce\"") 18 | public String getValueType(); 19 | 20 | @Config("timestamp_format") 21 | @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%6N\"") 22 | public String getTimestampFormat(); 23 | 24 | @Config("timezone") 25 | @ConfigDefault("null") 26 | public Optional getTimeZone(); 27 | } 28 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/JdbcOutputConnector.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.sql.SQLException; 4 | 5 | public interface JdbcOutputConnector 6 | { 7 | public JdbcOutputConnection connect(boolean autoCommit) throws SQLException; 8 | } 9 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/JdbcSchema.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Collections; 5 | import java.util.List; 6 | import java.util.Optional; 7 | 8 | import org.embulk.config.ConfigException; 9 | 10 | import com.fasterxml.jackson.annotation.JsonCreator; 11 | import com.fasterxml.jackson.annotation.JsonValue; 12 | 13 | public class JdbcSchema 14 | { 15 | private List columns; 16 | 17 | @JsonCreator 18 | public JdbcSchema(List columns) 19 | { 20 | this.columns = columns; 21 | } 22 | 23 | @JsonValue 24 | public List getColumns() 25 | { 26 | return columns; 27 | } 28 | 29 | public Optional findColumn(String name) 30 | { 31 | // because both upper case column and lower case column may exist, search twice 32 | for (JdbcColumn column : columns) { 33 | if (column.getName().equals(name)) { 34 | return Optional.of(column); 35 | } 36 | } 37 | 38 | JdbcColumn foundColumn = null; 39 | for (JdbcColumn column : columns) { 40 | if (column.getName().equalsIgnoreCase(name)) { 41 | if (foundColumn != null) { 42 | throw new ConfigException(String.format("Cannot specify column '%s' because both '%s' and '%s' exist.", 43 | name, foundColumn.getName(), column.getName())); 44 | } 45 | foundColumn = column; 46 | } 47 | } 48 | 49 | if (foundColumn != null) { 50 | return Optional.of(foundColumn); 51 | } 52 | return Optional.empty(); 53 | } 54 | 55 | public int getCount() 56 | { 57 | return columns.size(); 58 | } 59 | 60 | public JdbcColumn getColumn(int i) 61 | { 62 | return columns.get(i); 63 | } 64 | 65 | public String getColumnName(int i) 66 | { 67 | return columns.get(i).getName(); 68 | } 69 | 70 | public static JdbcSchema filterSkipColumns(JdbcSchema schema) 71 | { 72 | final ArrayList builder = new ArrayList<>(); 73 | for (JdbcColumn c : schema.getColumns()) { 74 | if (!c.isSkipColumn()) { 75 | builder.add(c); 76 | } 77 | } 78 | return new JdbcSchema(Collections.unmodifiableList(builder)); 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/MemoryRecord.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.time.Instant; 4 | import org.embulk.spi.Column; 5 | import org.msgpack.value.Value; 6 | 7 | public class MemoryRecord implements Record 8 | { 9 | private final Object[] values; 10 | 11 | public MemoryRecord(int columnCount) 12 | { 13 | values = new Object[columnCount]; 14 | } 15 | 16 | 17 | public boolean isNull(Column column) 18 | { 19 | return getValue(column) == null; 20 | } 21 | 22 | public boolean getBoolean(Column column) 23 | { 24 | return (Boolean)getValue(column); 25 | } 26 | 27 | public long getLong(Column column) 28 | { 29 | return (Long)getValue(column); 30 | } 31 | 32 | public double getDouble(Column column) 33 | { 34 | return (Double)getValue(column); 35 | } 36 | 37 | public String getString(Column column) 38 | { 39 | return (String)getValue(column); 40 | } 41 | 42 | public Instant getTimestamp(Column column) 43 | { 44 | return (Instant)getValue(column); 45 | } 46 | 47 | public Value getJson(Column column) 48 | { 49 | return (Value)getValue(column); 50 | } 51 | 52 | private Object getValue(Column column) 53 | { 54 | return values[column.getIndex()]; 55 | } 56 | 57 | public void setValue(Column column, Object value) 58 | { 59 | values[column.getIndex()] = value; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/MergeConfig.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.util.List; 4 | import java.util.Optional; 5 | 6 | public class MergeConfig { 7 | private final List mergeKeys; 8 | private final Optional> mergeRule; 9 | 10 | public MergeConfig(List mergeKeys, Optional> mergeRule) { 11 | this.mergeKeys = mergeKeys; 12 | this.mergeRule = mergeRule; 13 | } 14 | 15 | public List getMergeKeys() { 16 | return mergeKeys; 17 | } 18 | 19 | public Optional> getMergeRule() { 20 | return mergeRule; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/PageReaderRecord.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.time.Instant; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import org.embulk.spi.Column; 8 | import org.embulk.spi.Page; 9 | import org.embulk.spi.PageReader; 10 | import org.msgpack.value.Value; 11 | 12 | /** 13 | * Record read by PageReader. 14 | * The class will save read records for retry. 15 | */ 16 | public class PageReaderRecord implements Record 17 | { 18 | private final PageReader pageReader; 19 | private final List readRecords; 20 | private MemoryRecord lastRecord; 21 | 22 | public PageReaderRecord(PageReader pageReader) 23 | { 24 | this.pageReader = pageReader; 25 | readRecords = new ArrayList<>(); 26 | } 27 | 28 | public void setPage(Page page) 29 | { 30 | pageReader.setPage(page); 31 | } 32 | 33 | public boolean nextRecord() 34 | { 35 | lastRecord = null; // lastRecord will be created in next `save` method execution. 36 | return pageReader.nextRecord(); 37 | } 38 | 39 | public boolean isNull(Column column) 40 | { 41 | return pageReader.isNull(column); 42 | } 43 | 44 | public boolean getBoolean(Column column) 45 | { 46 | return save(column, pageReader.getBoolean(column)); 47 | } 48 | 49 | public long getLong(Column column) 50 | { 51 | return save(column, pageReader.getLong(column)); 52 | } 53 | 54 | public double getDouble(Column column) 55 | { 56 | return save(column, pageReader.getDouble(column)); 57 | } 58 | 59 | public String getString(Column column) 60 | { 61 | return save(column, pageReader.getString(column)); 62 | } 63 | 64 | public Instant getTimestamp(Column column) 65 | { 66 | return save(column, pageReader.getTimestamp(column).getInstant()); 67 | } 68 | 69 | public Value getJson(Column column) 70 | { 71 | return save(column, pageReader.getJson(column)); 72 | } 73 | 74 | public List getReadRecords() 75 | { 76 | return readRecords; 77 | } 78 | 79 | public void clearReadRecords() 80 | { 81 | readRecords.clear(); 82 | lastRecord = null; 83 | } 84 | 85 | private T save(Column column, T value) 86 | { 87 | if (lastRecord == null) { 88 | lastRecord = new MemoryRecord(pageReader.getSchema().getColumnCount()); 89 | readRecords.add(lastRecord); 90 | } 91 | lastRecord.setValue(column, value); 92 | return value; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/Record.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.time.Instant; 4 | import org.embulk.spi.Column; 5 | import org.msgpack.value.Value; 6 | 7 | public interface Record 8 | { 9 | boolean isNull(Column column); 10 | 11 | boolean getBoolean(Column column); 12 | 13 | long getLong(Column column); 14 | 15 | double getDouble(Column column); 16 | 17 | String getString(Column column); 18 | 19 | Instant getTimestamp(Column column); 20 | 21 | Value getJson(Column column); 22 | } 23 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/Ssl.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import com.fasterxml.jackson.annotation.JsonCreator; 4 | import com.fasterxml.jackson.annotation.JsonValue; 5 | 6 | import org.embulk.config.ConfigException; 7 | 8 | public enum Ssl 9 | { 10 | ENABLE, 11 | DISABLE, 12 | VERIFY; 13 | 14 | @JsonValue 15 | @Override 16 | public String toString() 17 | { 18 | return this.name().toLowerCase(); 19 | } 20 | 21 | @JsonCreator 22 | public static Ssl fromString(String value) 23 | { 24 | switch(value) { 25 | case "enable": 26 | case "true": 27 | return ENABLE; 28 | case "disable": 29 | case "false": 30 | return DISABLE; 31 | case "verify": 32 | return VERIFY; 33 | default: 34 | throw new ConfigException(String.format("Unknown SSL value '%s'. Supported values are enable, true, disable, false or verify.", value)); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/TableIdentifier.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | 5 | 6 | public class TableIdentifier 7 | { 8 | private String database; 9 | private String schemaName; 10 | private String tableName; 11 | 12 | public TableIdentifier(String database, String schemaName, String tableName) 13 | { 14 | this.database = database; 15 | this.schemaName = schemaName; 16 | this.tableName = tableName; 17 | } 18 | 19 | public TableIdentifier() 20 | { 21 | } 22 | 23 | @JsonProperty 24 | public String getDatabase() { 25 | return database; 26 | } 27 | 28 | @JsonProperty 29 | public void setDatabase(String database) { 30 | this.database = database; 31 | } 32 | 33 | @JsonProperty 34 | public String getSchemaName() 35 | { 36 | return schemaName; 37 | } 38 | 39 | @JsonProperty 40 | public void setSchemaName(String schemaName) 41 | { 42 | this.schemaName = schemaName; 43 | } 44 | 45 | @JsonProperty 46 | public String getTableName() { 47 | return tableName; 48 | } 49 | 50 | @JsonProperty 51 | public void setTableName(String tableName) { 52 | this.tableName = tableName; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/TimestampFormat.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.sql.Timestamp; 4 | import java.text.FieldPosition; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Date; 7 | 8 | 9 | public class TimestampFormat extends SimpleDateFormat 10 | { 11 | 12 | private final int scale; 13 | 14 | public TimestampFormat(String pattern, int scale) 15 | { 16 | super(pattern); 17 | 18 | this.scale = scale; 19 | } 20 | 21 | @Override 22 | public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition pos) 23 | { 24 | StringBuffer buffer = super.format(date, toAppendTo, pos); 25 | if (scale > 0) { 26 | buffer.append('.'); 27 | String nanos = Integer.toString(((Timestamp)date).getNanos()); 28 | int zeros = Math.min(scale, 9 - nanos.length()); 29 | for (int i = 0; i < zeros; i++) { 30 | buffer.append('0'); 31 | } 32 | buffer.append(nanos.substring(0, scale - zeros)); 33 | } 34 | return buffer; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/ToString.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.fasterxml.jackson.databind.JsonMappingException; 5 | import com.fasterxml.jackson.databind.node.NullNode; 6 | import com.fasterxml.jackson.annotation.JsonCreator; 7 | import com.fasterxml.jackson.annotation.JsonValue; 8 | import java.util.Optional; 9 | 10 | public class ToString 11 | { 12 | private final String string; 13 | 14 | public ToString(String string) 15 | { 16 | this.string = string; 17 | } 18 | 19 | @JsonCreator 20 | ToString(Optional option) throws JsonMappingException 21 | { 22 | JsonNode node = option.orElse(NullNode.getInstance()); 23 | if (node.isTextual()) { 24 | this.string = node.textValue(); 25 | } else if (node.isValueNode()) { 26 | this.string = node.toString(); 27 | } else { 28 | throw new JsonMappingException(String.format("Arrays and objects are invalid: '%s'", node)); 29 | } 30 | } 31 | 32 | @Override 33 | public boolean equals(Object obj) 34 | { 35 | if (!(obj instanceof ToString)) { 36 | return false; 37 | } 38 | ToString o = (ToString) obj; 39 | return string.equals(o.string); 40 | } 41 | 42 | @Override 43 | public int hashCode() 44 | { 45 | return string.hashCode(); 46 | } 47 | 48 | @JsonValue 49 | @Override 50 | public String toString() 51 | { 52 | return string; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/ToStringMap.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.util.Map; 4 | import java.util.HashMap; 5 | import java.util.Properties; 6 | import com.fasterxml.jackson.annotation.JsonCreator; 7 | 8 | public class ToStringMap 9 | extends HashMap 10 | { 11 | @JsonCreator 12 | ToStringMap(Map map) 13 | { 14 | super(mapToStringString(map)); 15 | } 16 | 17 | public Properties toProperties() 18 | { 19 | Properties props = new Properties(); 20 | props.putAll(this); 21 | return props; 22 | } 23 | 24 | private static Map mapToStringString(final Map mapOfToString) { 25 | final HashMap result = new HashMap<>(); 26 | for (final Map.Entry entry : mapOfToString.entrySet()) { 27 | final ToString value = entry.getValue(); 28 | if (value == null) { 29 | result.put(entry.getKey(), "null"); 30 | } else { 31 | result.put(entry.getKey(), value.toString()); 32 | } 33 | } 34 | return result; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/TransactionIsolation.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.sql.Connection; 4 | import java.util.Locale; 5 | 6 | import org.embulk.config.ConfigException; 7 | 8 | import com.fasterxml.jackson.annotation.JsonCreator; 9 | import com.fasterxml.jackson.annotation.JsonValue; 10 | 11 | public enum TransactionIsolation { 12 | READ_UNCOMMITTED { 13 | @Override 14 | public int toInt() { 15 | return Connection.TRANSACTION_READ_UNCOMMITTED; 16 | } 17 | }, 18 | READ_COMMITTED { 19 | @Override 20 | public int toInt() { 21 | return Connection.TRANSACTION_READ_COMMITTED; 22 | } 23 | }, 24 | REPEATABLE_READ { 25 | @Override 26 | public int toInt() { 27 | return Connection.TRANSACTION_REPEATABLE_READ; 28 | } 29 | }, 30 | SERIALIZABLE { 31 | @Override 32 | public int toInt() { 33 | return Connection.TRANSACTION_SERIALIZABLE; 34 | } 35 | }; 36 | 37 | @JsonValue 38 | @Override 39 | public String toString() 40 | { 41 | return name().toLowerCase(Locale.ENGLISH); 42 | } 43 | 44 | public abstract int toInt(); 45 | 46 | @JsonCreator 47 | public static TransactionIsolation fromString(String value) 48 | { 49 | for (TransactionIsolation ti : values()) { 50 | if (ti.toString().equals(value)) { 51 | return ti; 52 | } 53 | } 54 | throw new ConfigException(String.format("Unknown transaction_isolation '%s'.", value)); 55 | } 56 | 57 | public static TransactionIsolation fromInt(int value) 58 | { 59 | for (TransactionIsolation ti : values()) { 60 | if (ti.toInt() == value) { 61 | return ti; 62 | } 63 | } 64 | throw new IllegalArgumentException(String.format("Unknown transaction_isolation '%d'.", value)); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/BigDecimalColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.math.BigDecimal; 4 | import java.io.IOException; 5 | import java.sql.SQLException; 6 | import java.time.Instant; 7 | 8 | import org.embulk.output.jdbc.JdbcColumn; 9 | import org.embulk.output.jdbc.BatchInsert; 10 | import org.msgpack.value.Value; 11 | 12 | public class BigDecimalColumnSetter 13 | extends ColumnSetter 14 | { 15 | private static final BigDecimal ZERO = BigDecimal.valueOf(0L); 16 | private static final BigDecimal ONE = BigDecimal.valueOf(1L); 17 | 18 | public BigDecimalColumnSetter(BatchInsert batch, JdbcColumn column, 19 | DefaultValueSetter defaultValue) 20 | { 21 | super(batch, column, defaultValue); 22 | } 23 | 24 | @Override 25 | public void nullValue() throws IOException, SQLException 26 | { 27 | defaultValue.setBigDecimal(); 28 | } 29 | 30 | @Override 31 | public void booleanValue(boolean v) throws IOException, SQLException 32 | { 33 | batch.setBigDecimal(v ? ONE : ZERO); 34 | } 35 | 36 | @Override 37 | public void longValue(long v) throws IOException, SQLException 38 | { 39 | batch.setBigDecimal(BigDecimal.valueOf(v)); 40 | } 41 | 42 | @Override 43 | public void doubleValue(double v) throws IOException, SQLException 44 | { 45 | if (Double.isNaN(v) || Double.isInfinite(v)) { 46 | defaultValue.setBigDecimal(); 47 | } else { 48 | batch.setBigDecimal(BigDecimal.valueOf(v)); 49 | } 50 | } 51 | 52 | @Override 53 | public void stringValue(String v) throws IOException, SQLException 54 | { 55 | BigDecimal dv; 56 | try { 57 | dv = new BigDecimal(v); 58 | } catch (NumberFormatException ex) { 59 | defaultValue.setBigDecimal(); 60 | return; 61 | } 62 | batch.setBigDecimal(dv); 63 | } 64 | 65 | @Override 66 | public void timestampValue(final Instant v) throws IOException, SQLException 67 | { 68 | defaultValue.setBigDecimal(); 69 | } 70 | 71 | @Override 72 | public void jsonValue(Value v) throws IOException, SQLException 73 | { 74 | defaultValue.setBigDecimal(); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/BooleanColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.time.Instant; 6 | 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class BooleanColumnSetter 12 | extends ColumnSetter 13 | { 14 | public BooleanColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void nullValue() throws IOException, SQLException 22 | { 23 | defaultValue.setBoolean(); 24 | } 25 | 26 | @Override 27 | public void booleanValue(boolean v) throws IOException, SQLException 28 | { 29 | batch.setBoolean(v); 30 | } 31 | 32 | @Override 33 | public void longValue(long v) throws IOException, SQLException 34 | { 35 | batch.setBoolean(v > 0); 36 | } 37 | 38 | @Override 39 | public void doubleValue(double v) throws IOException, SQLException 40 | { 41 | batch.setBoolean(v > 0.0); 42 | } 43 | 44 | @Override 45 | public void stringValue(String v) throws IOException, SQLException 46 | { 47 | defaultValue.setBoolean(); 48 | } 49 | 50 | @Override 51 | public void timestampValue(final Instant v) throws IOException, SQLException 52 | { 53 | defaultValue.setBoolean(); 54 | } 55 | 56 | @Override 57 | public void jsonValue(Value v) throws IOException, SQLException 58 | { 59 | defaultValue.setBoolean(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/ByteColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.math.RoundingMode; 6 | import java.time.Instant; 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class ByteColumnSetter 12 | extends ColumnSetter 13 | { 14 | public ByteColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void nullValue() throws IOException, SQLException 22 | { 23 | defaultValue.setByte(); 24 | } 25 | 26 | @Override 27 | public void booleanValue(boolean v) throws IOException, SQLException 28 | { 29 | batch.setByte(v ? (byte) 1 : (byte) 0); 30 | } 31 | 32 | @Override 33 | public void longValue(long v) throws IOException, SQLException 34 | { 35 | if (v > Byte.MAX_VALUE || v < Byte.MIN_VALUE) { 36 | defaultValue.setByte(); 37 | } else { 38 | batch.setByte((byte) v); 39 | } 40 | } 41 | 42 | @Override 43 | public void doubleValue(double v) throws IOException, SQLException 44 | { 45 | long lv; 46 | try { 47 | // TODO configurable rounding mode 48 | lv = roundDoubleToLong(v); 49 | } catch (ArithmeticException ex) { 50 | // NaN / Infinite / -Infinite 51 | defaultValue.setByte(); 52 | return; 53 | } 54 | longValue(lv); 55 | } 56 | 57 | @Override 58 | public void stringValue(String v) throws IOException, SQLException 59 | { 60 | byte sv; 61 | try { 62 | sv = Byte.parseByte(v); 63 | } catch (NumberFormatException e) { 64 | defaultValue.setByte(); 65 | return; 66 | } 67 | batch.setByte(sv); 68 | } 69 | 70 | @Override 71 | public void timestampValue(final Instant v) throws IOException, SQLException 72 | { 73 | defaultValue.setByte(); 74 | } 75 | 76 | @Override 77 | public void jsonValue(Value v) throws IOException, SQLException 78 | { 79 | defaultValue.setByte(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/ColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.time.Instant; 6 | 7 | import org.embulk.output.jdbc.BatchInsert; 8 | import org.embulk.output.jdbc.JdbcColumn; 9 | import org.msgpack.value.Value; 10 | 11 | public abstract class ColumnSetter 12 | { 13 | protected final BatchInsert batch; 14 | protected final JdbcColumn column; 15 | protected final DefaultValueSetter defaultValue; 16 | 17 | public ColumnSetter(BatchInsert batch, JdbcColumn column, 18 | DefaultValueSetter defaultValue) 19 | { 20 | this.batch = batch; 21 | this.column = column; 22 | this.defaultValue = defaultValue; 23 | } 24 | 25 | public JdbcColumn getColumn() 26 | { 27 | return column; 28 | } 29 | 30 | public int getSqlType() 31 | { 32 | return column.getSqlType(); 33 | } 34 | 35 | public abstract void nullValue() throws IOException, SQLException; 36 | 37 | public abstract void booleanValue(boolean v) throws IOException, SQLException; 38 | 39 | public abstract void longValue(long v) throws IOException, SQLException; 40 | 41 | public abstract void doubleValue(double v) throws IOException, SQLException; 42 | 43 | public abstract void stringValue(String v) throws IOException, SQLException; 44 | 45 | public abstract void timestampValue(final Instant v) throws IOException, SQLException; 46 | 47 | public abstract void jsonValue(Value v) throws IOException, SQLException; 48 | 49 | final long roundDoubleToLong(final double v) { 50 | if (Math.getExponent(v) > Double.MAX_EXPONENT) { 51 | throw new ArithmeticException("input is infinite or NaN"); 52 | } 53 | final double z = Math.rint(v); 54 | final double result; 55 | if (Math.abs(v - z) == 0.5) { 56 | result = v + Math.copySign(0.5, v); 57 | } else { 58 | result = z; 59 | } 60 | 61 | // What we want to confirm here is, in essense : 62 | // 63 | // result > ((double) Long.MIN_VALUE) - 1.0 && result < ((double) Long.MAX_VALUE) + 1.0 64 | // 65 | // However, |Long.MAX_VALUE| cannot be stored as a double without losing precision. 66 | // Instead, |Long.MAX_VALUE + 1| can be stored as a double. 67 | // 68 | // Then, we use |-0x1p63| for |Long.MIN_VALUE|, and |0x1p63| for |Long.MAX_VALUE + 1|, such as : 69 | // 70 | // result > ((double) -0x1p63) - 1.0 && result < ((double) 0x1p63) 71 | // 72 | // Then, the inequation is transformed into : 73 | // 74 | // result - ((double) -0x1p63) > -1.0 && result < ((double) 0x1p63) 75 | // 76 | // And then : 77 | // 78 | // ((double) -0x1p63) - result < 1.0 && result < ((double) 0x1p63) 79 | if (!(((double) -0x1p63) - result < 1.0 && result < ((double) 0x1p63))) { 80 | throw new ArithmeticException("not in range"); 81 | } 82 | 83 | return (long) result; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/ColumnSetterVisitor.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import org.embulk.output.jdbc.Record; 7 | import org.embulk.spi.Column; 8 | import org.embulk.spi.ColumnVisitor; 9 | 10 | public class ColumnSetterVisitor 11 | implements ColumnVisitor 12 | { 13 | private final Record record; 14 | private final ColumnSetter setter; 15 | 16 | public ColumnSetterVisitor(Record record, ColumnSetter setter) 17 | { 18 | this.record = record; 19 | this.setter = setter; 20 | } 21 | 22 | @Override 23 | public void booleanColumn(Column column) 24 | { 25 | try { 26 | if (record.isNull(column)) { 27 | setter.nullValue(); 28 | } else { 29 | setter.booleanValue(record.getBoolean(column)); 30 | } 31 | } catch (IOException | SQLException ex) { 32 | // TODO exception class 33 | throw new RuntimeException(ex); 34 | } 35 | } 36 | 37 | @Override 38 | public void longColumn(Column column) 39 | { 40 | try { 41 | if (record.isNull(column)) { 42 | setter.nullValue(); 43 | } else { 44 | setter.longValue(record.getLong(column)); 45 | } 46 | } catch (IOException | SQLException ex) { 47 | // TODO exception class 48 | throw new RuntimeException(ex); 49 | } 50 | } 51 | 52 | @Override 53 | public void doubleColumn(Column column) 54 | { 55 | try { 56 | if (record.isNull(column)) { 57 | setter.nullValue(); 58 | } else { 59 | setter.doubleValue(record.getDouble(column)); 60 | } 61 | } catch (IOException | SQLException ex) { 62 | // TODO exception class 63 | throw new RuntimeException(ex); 64 | } 65 | } 66 | 67 | @Override 68 | public void stringColumn(Column column) 69 | { 70 | try { 71 | if (record.isNull(column)) { 72 | setter.nullValue(); 73 | } else { 74 | setter.stringValue(record.getString(column)); 75 | } 76 | } catch (IOException | SQLException ex) { 77 | // TODO exception class 78 | throw new RuntimeException(ex); 79 | } 80 | } 81 | 82 | @Override 83 | public void jsonColumn(Column column) 84 | { 85 | try { 86 | if (record.isNull(column)) { 87 | setter.nullValue(); 88 | } else { 89 | setter.jsonValue(record.getJson(column)); 90 | } 91 | } catch (IOException | SQLException ex) { 92 | // TODO exception class 93 | throw new RuntimeException(ex); 94 | } 95 | } 96 | 97 | @Override 98 | public void timestampColumn(Column column) 99 | { 100 | try { 101 | if (record.isNull(column)) { 102 | setter.nullValue(); 103 | } else { 104 | setter.timestampValue(record.getTimestamp(column)); 105 | } 106 | } catch (IOException | SQLException ex) { 107 | // TODO exception class 108 | throw new RuntimeException(ex); 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/DefaultValueSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import org.embulk.output.jdbc.JdbcColumn; 6 | import org.embulk.output.jdbc.BatchInsert; 7 | 8 | public abstract class DefaultValueSetter 9 | { 10 | protected final BatchInsert batch; 11 | protected final JdbcColumn column; 12 | 13 | public DefaultValueSetter(BatchInsert batch, JdbcColumn column) 14 | { 15 | this.batch = batch; 16 | this.column = column; 17 | } 18 | 19 | public abstract void setNull() throws IOException, SQLException; 20 | 21 | public abstract void setBoolean() throws IOException, SQLException; 22 | 23 | public abstract void setByte() throws IOException, SQLException; 24 | 25 | public abstract void setShort() throws IOException, SQLException; 26 | 27 | public abstract void setInt() throws IOException, SQLException; 28 | 29 | public abstract void setLong() throws IOException, SQLException; 30 | 31 | public abstract void setFloat() throws IOException, SQLException; 32 | 33 | public abstract void setDouble() throws IOException, SQLException; 34 | 35 | public abstract void setBigDecimal() throws IOException, SQLException; 36 | 37 | public abstract void setString() throws IOException, SQLException; 38 | 39 | public abstract void setNString() throws IOException, SQLException; 40 | 41 | public abstract void setBytes() throws IOException, SQLException; 42 | 43 | public abstract void setSqlDate() throws IOException, SQLException; 44 | 45 | public abstract void setSqlTime() throws IOException, SQLException; 46 | 47 | public abstract void setSqlTimestamp() throws IOException, SQLException; 48 | 49 | public abstract void setJson() throws IOException, SQLException; 50 | } 51 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/DoubleColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.time.Instant; 6 | 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class DoubleColumnSetter 12 | extends ColumnSetter 13 | { 14 | public DoubleColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void nullValue() throws IOException, SQLException 22 | { 23 | defaultValue.setDouble(); 24 | } 25 | 26 | @Override 27 | public void booleanValue(boolean v) throws IOException, SQLException 28 | { 29 | batch.setDouble(v ? 1.0 : 0.0); 30 | } 31 | 32 | @Override 33 | public void longValue(long v) throws IOException, SQLException 34 | { 35 | batch.setDouble((double) v); 36 | } 37 | 38 | @Override 39 | public void doubleValue(double v) throws IOException, SQLException 40 | { 41 | batch.setDouble(v); 42 | } 43 | 44 | @Override 45 | public void stringValue(String v) throws IOException, SQLException 46 | { 47 | double dv; 48 | try { 49 | dv = Double.parseDouble(v); 50 | } catch (NumberFormatException e) { 51 | defaultValue.setDouble(); 52 | return; 53 | } 54 | batch.setDouble(dv); 55 | } 56 | 57 | @Override 58 | public void timestampValue(final Instant v) throws IOException, SQLException 59 | { 60 | defaultValue.setDouble(); 61 | } 62 | 63 | @Override 64 | public void jsonValue(Value v) throws IOException, SQLException 65 | { 66 | defaultValue.setDouble(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/FloatColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.time.Instant; 6 | 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class FloatColumnSetter 12 | extends ColumnSetter 13 | { 14 | public FloatColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void nullValue() throws IOException, SQLException 22 | { 23 | defaultValue.setFloat(); 24 | } 25 | 26 | @Override 27 | public void booleanValue(boolean v) throws IOException, SQLException 28 | { 29 | batch.setFloat(v ? (float) 1.0 : (float) 0.0); 30 | } 31 | 32 | @Override 33 | public void longValue(long v) throws IOException, SQLException 34 | { 35 | batch.setFloat((float) v); 36 | } 37 | 38 | @Override 39 | public void doubleValue(double v) throws IOException, SQLException 40 | { 41 | batch.setFloat((float) v); 42 | } 43 | 44 | @Override 45 | public void stringValue(String v) throws IOException, SQLException 46 | { 47 | float fv; 48 | try { 49 | fv = Float.parseFloat(v); 50 | } catch (NumberFormatException e) { 51 | defaultValue.setFloat(); 52 | return; 53 | } 54 | batch.setFloat(fv); 55 | } 56 | 57 | @Override 58 | public void timestampValue(final Instant v) throws IOException, SQLException 59 | { 60 | defaultValue.setFloat(); 61 | } 62 | 63 | @Override 64 | public void jsonValue(Value v) throws IOException, SQLException 65 | { 66 | defaultValue.setFloat(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/IntColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.math.RoundingMode; 6 | import java.time.Instant; 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class IntColumnSetter 12 | extends ColumnSetter 13 | { 14 | public IntColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void nullValue() throws IOException, SQLException 22 | { 23 | defaultValue.setInt(); 24 | } 25 | 26 | @Override 27 | public void booleanValue(boolean v) throws IOException, SQLException 28 | { 29 | batch.setInt(v ? 1 : 0); 30 | } 31 | 32 | @Override 33 | public void longValue(long v) throws IOException, SQLException 34 | { 35 | if (v > Integer.MAX_VALUE || v < Integer.MIN_VALUE) { 36 | defaultValue.setInt(); 37 | } else { 38 | batch.setInt((int) v); 39 | } 40 | } 41 | 42 | @Override 43 | public void doubleValue(double v) throws IOException, SQLException 44 | { 45 | long lv; 46 | try { 47 | // TODO configurable rounding mode 48 | lv = roundDoubleToLong(v); 49 | } catch (ArithmeticException ex) { 50 | // NaN / Infinite / -Infinite 51 | defaultValue.setInt(); 52 | return; 53 | } 54 | longValue(lv); 55 | } 56 | 57 | @Override 58 | public void stringValue(String v) throws IOException, SQLException 59 | { 60 | int iv; 61 | try { 62 | iv = Integer.parseInt(v); 63 | } catch (NumberFormatException e) { 64 | defaultValue.setInt(); 65 | return; 66 | } 67 | batch.setInt(iv); 68 | } 69 | 70 | @Override 71 | public void timestampValue(final Instant v) throws IOException, SQLException 72 | { 73 | defaultValue.setInt(); 74 | } 75 | 76 | @Override 77 | public void jsonValue(Value v) throws IOException, SQLException 78 | { 79 | defaultValue.setInt(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/JsonColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.time.Instant; 6 | 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class JsonColumnSetter 12 | extends ColumnSetter 13 | { 14 | public JsonColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void nullValue() throws IOException, SQLException 22 | { 23 | defaultValue.setJson(); 24 | } 25 | 26 | @Override 27 | public void booleanValue(boolean v) throws IOException, SQLException 28 | { 29 | defaultValue.setJson(); 30 | } 31 | 32 | @Override 33 | public void longValue(long v) throws IOException, SQLException 34 | { 35 | defaultValue.setJson(); 36 | } 37 | 38 | @Override 39 | public void doubleValue(double v) throws IOException, SQLException 40 | { 41 | defaultValue.setJson(); 42 | } 43 | 44 | @Override 45 | public void stringValue(String v) throws IOException, SQLException 46 | { 47 | defaultValue.setJson(); 48 | } 49 | 50 | @Override 51 | public void timestampValue(final Instant v) throws IOException, SQLException 52 | { 53 | defaultValue.setJson(); 54 | } 55 | 56 | @Override 57 | public void jsonValue(Value v) throws IOException, SQLException { 58 | batch.setString(v.toJson()); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/LongColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.math.RoundingMode; 6 | import java.time.Instant; 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class LongColumnSetter 12 | extends ColumnSetter 13 | { 14 | public LongColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void nullValue() throws IOException, SQLException 22 | { 23 | defaultValue.setLong(); 24 | } 25 | 26 | @Override 27 | public void booleanValue(boolean v) throws IOException, SQLException 28 | { 29 | batch.setLong(v ? 1L : 0L); 30 | } 31 | 32 | @Override 33 | public void longValue(long v) throws IOException, SQLException 34 | { 35 | batch.setLong(v); 36 | } 37 | 38 | @Override 39 | public void doubleValue(double v) throws IOException, SQLException 40 | { 41 | long lv; 42 | try { 43 | // TODO configurable rounding mode 44 | lv = roundDoubleToLong(v); 45 | } catch (ArithmeticException ex) { 46 | // NaN / Infinite / -Infinite 47 | defaultValue.setLong(); 48 | return; 49 | } 50 | batch.setLong(lv); 51 | } 52 | 53 | @Override 54 | public void stringValue(String v) throws IOException, SQLException 55 | { 56 | long lv; 57 | try { 58 | lv = Long.parseLong(v); 59 | } catch (NumberFormatException e) { 60 | defaultValue.setLong(); 61 | return; 62 | } 63 | batch.setLong(lv); 64 | } 65 | 66 | @Override 67 | public void timestampValue(final Instant v) throws IOException, SQLException 68 | { 69 | defaultValue.setLong(); 70 | } 71 | 72 | @Override 73 | public void jsonValue(Value v) throws IOException, SQLException 74 | { 75 | defaultValue.setLong(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/NStringColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.time.Instant; 6 | 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.embulk.util.timestamp.TimestampFormatter; 10 | import org.msgpack.value.Value; 11 | 12 | public class NStringColumnSetter 13 | extends ColumnSetter 14 | { 15 | private final TimestampFormatter timestampFormatter; 16 | 17 | public NStringColumnSetter(BatchInsert batch, JdbcColumn column, 18 | DefaultValueSetter defaultValue, 19 | TimestampFormatter timestampFormatter) 20 | { 21 | super(batch, column, defaultValue); 22 | this.timestampFormatter = timestampFormatter; 23 | } 24 | 25 | @Override 26 | public void nullValue() throws IOException, SQLException 27 | { 28 | defaultValue.setNString(); 29 | } 30 | 31 | @Override 32 | public void booleanValue(boolean v) throws IOException, SQLException 33 | { 34 | batch.setNString(Boolean.toString(v)); 35 | } 36 | 37 | @Override 38 | public void longValue(long v) throws IOException, SQLException 39 | { 40 | batch.setNString(Long.toString(v)); 41 | } 42 | 43 | @Override 44 | public void doubleValue(double v) throws IOException, SQLException 45 | { 46 | batch.setNString(Double.toString(v)); 47 | } 48 | 49 | @Override 50 | public void stringValue(String v) throws IOException, SQLException 51 | { 52 | batch.setNString(v); 53 | } 54 | 55 | @Override 56 | public void timestampValue(final Instant v) throws IOException, SQLException 57 | { 58 | batch.setNString(timestampFormatter.format(v)); 59 | } 60 | 61 | @Override 62 | public void jsonValue(Value v) throws IOException, SQLException 63 | { 64 | batch.setNString(v.toJson()); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/NullColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.time.Instant; 6 | 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class NullColumnSetter 12 | extends ColumnSetter 13 | { 14 | public NullColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void booleanValue(boolean v) throws IOException, SQLException 22 | { 23 | defaultValue.setNull(); 24 | } 25 | 26 | @Override 27 | public void longValue(long v) throws IOException, SQLException 28 | { 29 | defaultValue.setNull(); 30 | } 31 | 32 | @Override 33 | public void doubleValue(double v) throws IOException, SQLException 34 | { 35 | defaultValue.setNull(); 36 | } 37 | 38 | @Override 39 | public void stringValue(String v) throws IOException, SQLException 40 | { 41 | defaultValue.setNull(); 42 | } 43 | 44 | @Override 45 | public void timestampValue(final Instant v) throws IOException, SQLException 46 | { 47 | defaultValue.setNull(); 48 | } 49 | 50 | @Override 51 | public void nullValue() throws IOException, SQLException 52 | { 53 | defaultValue.setNull(); 54 | } 55 | 56 | @Override 57 | public void jsonValue(Value v) throws IOException, SQLException 58 | { 59 | defaultValue.setNull(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/NullDefaultValueSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import org.embulk.output.jdbc.JdbcColumn; 6 | import org.embulk.output.jdbc.BatchInsert; 7 | 8 | public class NullDefaultValueSetter 9 | extends DefaultValueSetter 10 | { 11 | public NullDefaultValueSetter(BatchInsert batch, JdbcColumn column) 12 | { 13 | super(batch, column); 14 | } 15 | 16 | @Override 17 | public void setNull() throws IOException, SQLException 18 | { 19 | batch.setNull(column.getSqlType()); 20 | } 21 | 22 | @Override 23 | public void setBoolean() throws IOException, SQLException 24 | { 25 | batch.setNull(column.getSqlType()); 26 | } 27 | 28 | @Override 29 | public void setByte() throws IOException, SQLException 30 | { 31 | batch.setNull(column.getSqlType()); 32 | } 33 | 34 | @Override 35 | public void setShort() throws IOException, SQLException 36 | { 37 | batch.setNull(column.getSqlType()); 38 | } 39 | 40 | @Override 41 | public void setInt() throws IOException, SQLException 42 | { 43 | batch.setNull(column.getSqlType()); 44 | } 45 | 46 | @Override 47 | public void setLong() throws IOException, SQLException 48 | { 49 | batch.setNull(column.getSqlType()); 50 | } 51 | 52 | @Override 53 | public void setFloat() throws IOException, SQLException 54 | { 55 | batch.setNull(column.getSqlType()); 56 | } 57 | 58 | @Override 59 | public void setDouble() throws IOException, SQLException 60 | { 61 | batch.setNull(column.getSqlType()); 62 | } 63 | 64 | @Override 65 | public void setBigDecimal() throws IOException, SQLException 66 | { 67 | batch.setNull(column.getSqlType()); 68 | } 69 | 70 | @Override 71 | public void setString() throws IOException, SQLException 72 | { 73 | batch.setNull(column.getSqlType()); 74 | } 75 | 76 | @Override 77 | public void setNString() throws IOException, SQLException 78 | { 79 | batch.setNull(column.getSqlType()); 80 | } 81 | 82 | @Override 83 | public void setBytes() throws IOException, SQLException 84 | { 85 | batch.setNull(column.getSqlType()); 86 | } 87 | 88 | @Override 89 | public void setSqlDate() throws IOException, SQLException 90 | { 91 | batch.setNull(column.getSqlType()); 92 | } 93 | 94 | @Override 95 | public void setSqlTime() throws IOException, SQLException 96 | { 97 | batch.setNull(column.getSqlType()); 98 | } 99 | 100 | @Override 101 | public void setSqlTimestamp() throws IOException, SQLException 102 | { 103 | batch.setNull(column.getSqlType()); 104 | } 105 | 106 | @Override 107 | public void setJson() throws IOException, SQLException 108 | { 109 | batch.setNull(column.getSqlType()); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/PassThroughColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.util.Calendar; 4 | import java.io.IOException; 5 | import java.sql.SQLException; 6 | import java.time.Instant; 7 | 8 | import org.embulk.output.jdbc.JdbcColumn; 9 | import org.embulk.output.jdbc.BatchInsert; 10 | import org.msgpack.value.Value; 11 | 12 | public class PassThroughColumnSetter 13 | extends ColumnSetter 14 | { 15 | private final Calendar calendar; 16 | 17 | public PassThroughColumnSetter(BatchInsert batch, JdbcColumn column, 18 | DefaultValueSetter defaultValue, 19 | Calendar calendar) 20 | { 21 | super(batch, column, defaultValue); 22 | this.calendar = calendar; 23 | } 24 | 25 | @Override 26 | public void nullValue() throws IOException, SQLException 27 | { 28 | batch.setNull(column.getSqlType()); 29 | } 30 | 31 | @Override 32 | public void booleanValue(boolean v) throws IOException, SQLException 33 | { 34 | batch.setBoolean(v); 35 | } 36 | 37 | @Override 38 | public void longValue(long v) throws IOException, SQLException 39 | { 40 | batch.setLong(v); 41 | } 42 | 43 | @Override 44 | public void doubleValue(double v) throws IOException, SQLException 45 | { 46 | batch.setDouble(v); 47 | } 48 | 49 | @Override 50 | public void stringValue(String v) throws IOException, SQLException 51 | { 52 | batch.setString(v); 53 | } 54 | 55 | @Override 56 | public void timestampValue(final Instant v) throws IOException, SQLException 57 | { 58 | batch.setSqlTimestamp(v, calendar); 59 | } 60 | 61 | @Override 62 | public void jsonValue(Value v) throws IOException, SQLException 63 | { 64 | batch.setString(v.toJson()); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/ShortColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.math.RoundingMode; 6 | import java.time.Instant; 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.msgpack.value.Value; 10 | 11 | public class ShortColumnSetter 12 | extends ColumnSetter 13 | { 14 | public ShortColumnSetter(BatchInsert batch, JdbcColumn column, 15 | DefaultValueSetter defaultValue) 16 | { 17 | super(batch, column, defaultValue); 18 | } 19 | 20 | @Override 21 | public void nullValue() throws IOException, SQLException 22 | { 23 | defaultValue.setShort(); 24 | } 25 | 26 | @Override 27 | public void booleanValue(boolean v) throws IOException, SQLException 28 | { 29 | batch.setShort(v ? (short) 1 : (short) 0); 30 | } 31 | 32 | @Override 33 | public void longValue(long v) throws IOException, SQLException 34 | { 35 | if (v > Short.MAX_VALUE || v < Short.MIN_VALUE) { 36 | defaultValue.setShort(); 37 | } else { 38 | batch.setShort((short) v); 39 | } 40 | } 41 | 42 | @Override 43 | public void doubleValue(double v) throws IOException, SQLException 44 | { 45 | long lv; 46 | try { 47 | // TODO configurable rounding mode 48 | lv = roundDoubleToLong(v); 49 | } catch (ArithmeticException ex) { 50 | // NaN / Infinite / -Infinite 51 | defaultValue.setShort(); 52 | return; 53 | } 54 | longValue(lv); 55 | } 56 | 57 | @Override 58 | public void stringValue(String v) throws IOException, SQLException 59 | { 60 | short sv; 61 | try { 62 | sv = Short.parseShort(v); 63 | } catch (NumberFormatException e) { 64 | defaultValue.setShort(); 65 | return; 66 | } 67 | batch.setShort(sv); 68 | } 69 | 70 | @Override 71 | public void timestampValue(final Instant v) throws IOException, SQLException 72 | { 73 | defaultValue.setShort(); 74 | } 75 | 76 | @Override 77 | public void jsonValue(Value v) throws IOException, SQLException 78 | { 79 | defaultValue.setShort(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/SkipColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.time.Instant; 4 | import org.embulk.output.jdbc.BatchInsert; 5 | import org.msgpack.value.Value; 6 | 7 | public class SkipColumnSetter 8 | extends ColumnSetter 9 | { 10 | public SkipColumnSetter(BatchInsert batch) 11 | { 12 | super(batch, null, null); 13 | } 14 | 15 | @Override 16 | public void booleanValue(boolean v) 17 | { 18 | } 19 | 20 | @Override 21 | public void longValue(long v) 22 | { 23 | } 24 | 25 | @Override 26 | public void doubleValue(double v) 27 | { 28 | } 29 | 30 | @Override 31 | public void stringValue(String v) 32 | { 33 | } 34 | 35 | @Override 36 | public void timestampValue(final Instant v) 37 | { 38 | } 39 | 40 | @Override 41 | public void nullValue() 42 | { 43 | } 44 | 45 | @Override 46 | public void jsonValue(Value v) 47 | { 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/SqlDateColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.util.Calendar; 4 | import java.io.IOException; 5 | import java.sql.SQLException; 6 | import java.time.Instant; 7 | 8 | import org.embulk.output.jdbc.JdbcColumn; 9 | import org.embulk.output.jdbc.BatchInsert; 10 | import org.msgpack.value.Value; 11 | 12 | public class SqlDateColumnSetter 13 | extends ColumnSetter 14 | { 15 | protected final Calendar calendar; 16 | 17 | public SqlDateColumnSetter(BatchInsert batch, JdbcColumn column, 18 | DefaultValueSetter defaultValue, 19 | Calendar calendar) 20 | { 21 | super(batch, column, defaultValue); 22 | this.calendar = calendar; 23 | } 24 | 25 | @Override 26 | public void nullValue() throws IOException, SQLException 27 | { 28 | defaultValue.setSqlDate(); 29 | } 30 | 31 | @Override 32 | public void booleanValue(boolean v) throws IOException, SQLException 33 | { 34 | defaultValue.setSqlDate(); 35 | } 36 | 37 | @Override 38 | public void longValue(long v) throws IOException, SQLException 39 | { 40 | defaultValue.setSqlDate(); 41 | } 42 | 43 | @Override 44 | public void doubleValue(double v) throws IOException, SQLException 45 | { 46 | defaultValue.setSqlDate(); 47 | } 48 | 49 | @Override 50 | public void stringValue(String v) throws IOException, SQLException 51 | { 52 | defaultValue.setSqlDate(); 53 | } 54 | 55 | @Override 56 | public void timestampValue(final Instant v) throws IOException, SQLException 57 | { 58 | batch.setSqlDate(v, calendar); 59 | } 60 | 61 | @Override 62 | public void jsonValue(Value v) throws IOException, SQLException 63 | { 64 | defaultValue.setSqlDate(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/SqlTimeColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.util.Calendar; 4 | import java.io.IOException; 5 | import java.sql.SQLException; 6 | import java.time.Instant; 7 | 8 | import org.embulk.output.jdbc.JdbcColumn; 9 | import org.embulk.output.jdbc.BatchInsert; 10 | import org.msgpack.value.Value; 11 | 12 | public class SqlTimeColumnSetter 13 | extends ColumnSetter 14 | { 15 | protected final Calendar calendar; 16 | 17 | public SqlTimeColumnSetter(BatchInsert batch, JdbcColumn column, 18 | DefaultValueSetter defaultValue, 19 | Calendar calendar) 20 | { 21 | super(batch, column, defaultValue); 22 | this.calendar = calendar; 23 | } 24 | 25 | @Override 26 | public void nullValue() throws IOException, SQLException 27 | { 28 | defaultValue.setSqlTime(); 29 | } 30 | 31 | @Override 32 | public void booleanValue(boolean v) throws IOException, SQLException 33 | { 34 | defaultValue.setSqlTime(); 35 | } 36 | 37 | @Override 38 | public void longValue(long v) throws IOException, SQLException 39 | { 40 | defaultValue.setSqlTime(); 41 | } 42 | 43 | @Override 44 | public void doubleValue(double v) throws IOException, SQLException 45 | { 46 | defaultValue.setSqlTime(); 47 | } 48 | 49 | @Override 50 | public void stringValue(String v) throws IOException, SQLException 51 | { 52 | defaultValue.setSqlTime(); 53 | } 54 | 55 | @Override 56 | public void timestampValue(final Instant v) throws IOException, SQLException 57 | { 58 | batch.setSqlTime(v, calendar); 59 | } 60 | 61 | @Override 62 | public void jsonValue(Value v) throws IOException, SQLException 63 | { 64 | defaultValue.setSqlTime(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.util.Calendar; 4 | import java.io.IOException; 5 | import java.sql.SQLException; 6 | import java.time.Instant; 7 | 8 | import org.embulk.output.jdbc.JdbcColumn; 9 | import org.embulk.output.jdbc.BatchInsert; 10 | import org.msgpack.value.Value; 11 | 12 | public class SqlTimestampColumnSetter 13 | extends ColumnSetter 14 | { 15 | protected final Calendar calendar; 16 | 17 | public SqlTimestampColumnSetter(BatchInsert batch, JdbcColumn column, 18 | DefaultValueSetter defaultValue, 19 | Calendar calendar) 20 | { 21 | super(batch, column, defaultValue); 22 | this.calendar = calendar; 23 | } 24 | 25 | @Override 26 | public void nullValue() throws IOException, SQLException 27 | { 28 | defaultValue.setSqlTimestamp(); 29 | } 30 | 31 | @Override 32 | public void booleanValue(boolean v) throws IOException, SQLException 33 | { 34 | defaultValue.setSqlTimestamp(); 35 | } 36 | 37 | @Override 38 | public void longValue(long v) throws IOException, SQLException 39 | { 40 | defaultValue.setSqlTimestamp(); 41 | } 42 | 43 | @Override 44 | public void doubleValue(double v) throws IOException, SQLException 45 | { 46 | defaultValue.setSqlTimestamp(); 47 | } 48 | 49 | @Override 50 | public void stringValue(String v) throws IOException, SQLException 51 | { 52 | defaultValue.setSqlTimestamp(); 53 | } 54 | 55 | @Override 56 | public void timestampValue(final Instant v) throws IOException, SQLException 57 | { 58 | batch.setSqlTimestamp(v, calendar); 59 | } 60 | 61 | @Override 62 | public void jsonValue(Value v) throws IOException, SQLException 63 | { 64 | defaultValue.setSqlTimestamp(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.time.Instant; 6 | 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.embulk.util.timestamp.TimestampFormatter; 10 | import org.msgpack.value.Value; 11 | 12 | public class StringColumnSetter 13 | extends ColumnSetter 14 | { 15 | private final TimestampFormatter timestampFormatter; 16 | 17 | public StringColumnSetter(BatchInsert batch, JdbcColumn column, 18 | DefaultValueSetter defaultValue, 19 | TimestampFormatter timestampFormatter) 20 | { 21 | super(batch, column, defaultValue); 22 | this.timestampFormatter = timestampFormatter; 23 | } 24 | 25 | @Override 26 | public void nullValue() throws IOException, SQLException 27 | { 28 | defaultValue.setString(); 29 | } 30 | 31 | @Override 32 | public void booleanValue(boolean v) throws IOException, SQLException 33 | { 34 | batch.setString(Boolean.toString(v)); 35 | } 36 | 37 | @Override 38 | public void longValue(long v) throws IOException, SQLException 39 | { 40 | batch.setString(Long.toString(v)); 41 | } 42 | 43 | @Override 44 | public void doubleValue(double v) throws IOException, SQLException 45 | { 46 | batch.setString(Double.toString(v)); 47 | } 48 | 49 | @Override 50 | public void stringValue(String v) throws IOException, SQLException 51 | { 52 | batch.setString(v); 53 | } 54 | 55 | @Override 56 | public void timestampValue(final Instant v) throws IOException, SQLException 57 | { 58 | batch.setString(timestampFormatter.format(v)); 59 | } 60 | 61 | @Override 62 | public void jsonValue(Value v) throws IOException, SQLException 63 | { 64 | batch.setString(v.toJson()); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/test/java/org/embulk/output/jdbc/TestJdbcOutputPlugin.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import static org.junit.Assert.assertEquals; 4 | 5 | import org.junit.Test; 6 | 7 | public class TestJdbcOutputPlugin 8 | { 9 | @Test 10 | public void testCalculateSuffixLength() 11 | { 12 | { 13 | try { 14 | AbstractJdbcOutputPlugin.calculateSuffixLength(-1); 15 | } catch (java.lang.AssertionError e) { 16 | assert(true); 17 | } 18 | } 19 | { 20 | try { 21 | AbstractJdbcOutputPlugin.calculateSuffixLength(0); 22 | } catch (java.lang.AssertionError e) { 23 | assert(true); 24 | } 25 | } 26 | { 27 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(1); 28 | assertEquals(3, i); 29 | } 30 | { 31 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(10); 32 | assertEquals(3, i); 33 | } 34 | { 35 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(11); 36 | assertEquals(3, i); 37 | } 38 | { 39 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(100); 40 | assertEquals(3, i); 41 | } 42 | { 43 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(101); 44 | assertEquals(3, i); 45 | } 46 | { 47 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(1000); 48 | assertEquals(3, i); 49 | } 50 | { 51 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(1001); 52 | assertEquals(4, i); 53 | } 54 | { 55 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(10000); 56 | assertEquals(4, i); 57 | } 58 | { 59 | int i = AbstractJdbcOutputPlugin.calculateSuffixLength(10001); 60 | assertEquals(5, i); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /embulk-output-jdbc/src/test/java/org/embulk/output/jdbc/TimestampFormatTest.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.jdbc; 2 | 3 | import java.sql.Timestamp; 4 | import java.text.ParseException; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Date; 7 | 8 | import org.embulk.output.jdbc.TimestampFormat; 9 | import org.junit.Test; 10 | 11 | import static org.junit.Assert.assertEquals; 12 | 13 | public class TimestampFormatTest { 14 | 15 | @Test 16 | public void test() throws ParseException 17 | { 18 | Date date = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").parse("2015/03/04 17:08:09"); 19 | Timestamp t = new Timestamp(date.getTime()); 20 | 21 | { 22 | TimestampFormat format = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9); 23 | assertEquals("2015-03-04 17:08:09.000000000", format.format(t)); 24 | } 25 | { 26 | TimestampFormat format = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 0); 27 | assertEquals("2015-03-04 17:08:09", format.format(t)); 28 | } 29 | { 30 | TimestampFormat format = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 1); 31 | assertEquals("2015-03-04 17:08:09.0", format.format(t)); 32 | } 33 | 34 | t.setNanos(1234567); 35 | { 36 | TimestampFormat format = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9); 37 | assertEquals("2015-03-04 17:08:09.001234567", format.format(t)); 38 | } 39 | { 40 | TimestampFormat format = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 2); 41 | assertEquals("2015-03-04 17:08:09.00", format.format(t)); 42 | } 43 | { 44 | TimestampFormat format = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 3); 45 | assertEquals("2015-03-04 17:08:09.001", format.format(t)); 46 | } 47 | 48 | t.setNanos(123456789); 49 | { 50 | TimestampFormat format = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9); 51 | assertEquals("2015-03-04 17:08:09.123456789", format.format(t)); 52 | } 53 | { 54 | TimestampFormat format = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 1); 55 | assertEquals("2015-03-04 17:08:09.1", format.format(t)); 56 | } 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /embulk-output-mysql/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation(project(path: ":embulk-output-jdbc", configuration: "runtimeElements")) 3 | 4 | compileOnly "mysql:mysql-connector-java:5.1.44" 5 | defaultJdbcDriver 'mysql:mysql-connector-java:5.1.44' 6 | 7 | testImplementation "mysql:mysql-connector-java:5.1.44" 8 | 9 | testImplementation "org.embulk:embulk-formatter-csv:0.10.42" 10 | testImplementation "org.embulk:embulk-input-file:0.10.42" 11 | testImplementation "org.embulk:embulk-output-file:0.10.42" 12 | testImplementation "org.embulk:embulk-parser-csv:0.10.42" 13 | } 14 | 15 | embulkPlugin { 16 | mainClass = "org.embulk.output.MySQLOutputPlugin" 17 | category = "output" 18 | type = "mysql" 19 | } 20 | 21 | publishing { 22 | publications { 23 | maven(MavenPublication) { 24 | pom { // https://central.sonatype.org/pages/requirements.html 25 | developers { 26 | developer { 27 | name = "Sadayuki Furuhashi" 28 | email = "frsyuki@gmail.com" 29 | } 30 | developer { 31 | name = "takumakanari" 32 | email = "chemtrails.t@gmail.com" 33 | } 34 | developer { 35 | name = "Hitoshi Tanaka" 36 | email = "thitoshi@cac.co.jp" 37 | } 38 | developer { 39 | name = "Tomohiro Hashidate" 40 | email = "kakyoin.hierophant@gmail.com" 41 | } 42 | developer { 43 | name = "Hiroyuki Sato" 44 | email = "hiroysato@gmail.com" 45 | } 46 | developer { 47 | name = "Dai MIKURUBE" 48 | email = "dmikurube@treasure-data.com" 49 | } 50 | } 51 | } 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /embulk-output-mysql/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 9 | mysql:mysql-connector-java:5.1.44=compileClasspath 10 | org.embulk:embulk-spi:0.10.49=compileClasspath 11 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 12 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 13 | org.embulk:embulk-util-retryhelper:0.8.2=compileClasspath,runtimeClasspath 14 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 15 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 16 | org.msgpack:msgpack-core:0.8.24=compileClasspath 17 | org.slf4j:slf4j-api:2.0.7=compileClasspath 18 | empty= 19 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/main/java/org/embulk/output/mysql/MySQLBatchInsert.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.mysql; 2 | 3 | import java.io.IOException; 4 | import java.sql.Types; 5 | import java.sql.SQLException; 6 | import java.util.Optional; 7 | import org.embulk.output.jdbc.JdbcOutputConnector; 8 | import org.embulk.output.jdbc.MergeConfig; 9 | import org.embulk.output.jdbc.StandardBatchInsert; 10 | 11 | public class MySQLBatchInsert 12 | extends StandardBatchInsert 13 | { 14 | public MySQLBatchInsert(JdbcOutputConnector connector, Optional mergeConfig) throws IOException, SQLException 15 | { 16 | super(connector, mergeConfig); 17 | } 18 | 19 | @Override 20 | public void setFloat(float v) throws IOException, SQLException 21 | { 22 | if (Float.isNaN(v) || Float.isInfinite(v)) { 23 | setNull(Types.REAL); // TODO get through argument 24 | } else { 25 | super.setFloat(v); 26 | } 27 | } 28 | 29 | @Override 30 | public void setDouble(double v) throws IOException, SQLException 31 | { 32 | if (Double.isNaN(v) || Double.isInfinite(v)) { 33 | setNull(Types.DOUBLE); // TODO get through argument 34 | } else { 35 | super.setDouble(v); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/main/java/org/embulk/output/mysql/MySQLOutputConnector.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.mysql; 2 | 3 | import java.util.Properties; 4 | import java.sql.Connection; 5 | import java.sql.DriverManager; 6 | import java.sql.SQLException; 7 | import java.util.Optional; 8 | 9 | import org.embulk.output.jdbc.JdbcOutputConnection; 10 | import org.embulk.output.jdbc.AbstractJdbcOutputConnector; 11 | import org.embulk.output.jdbc.TransactionIsolation; 12 | 13 | public class MySQLOutputConnector 14 | extends AbstractJdbcOutputConnector 15 | { 16 | private final String url; 17 | private final Properties properties; 18 | 19 | public MySQLOutputConnector(String url, Properties properties, 20 | Optional transactionIsolation) 21 | { 22 | super(transactionIsolation); 23 | this.url = url; 24 | this.properties = properties; 25 | } 26 | 27 | @Override 28 | protected JdbcOutputConnection connect() throws SQLException 29 | { 30 | Connection c = DriverManager.getConnection(url, properties); 31 | try { 32 | MySQLOutputConnection con = new MySQLOutputConnection(c); 33 | c = null; 34 | return con; 35 | } finally { 36 | if (c != null) { 37 | c.close(); 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/java/org/embulk/output/mysql/CreateTableTest.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.mysql; 2 | 3 | import static org.embulk.output.mysql.MySQLTests.execute; 4 | import static org.embulk.output.mysql.MySQLTests.selectRecords; 5 | import static org.hamcrest.Matchers.is; 6 | import static org.hamcrest.MatcherAssert.assertThat; 7 | 8 | import java.io.File; 9 | import java.net.URISyntaxException; 10 | import java.net.URL; 11 | import java.nio.file.FileSystems; 12 | import java.nio.file.Path; 13 | import java.util.Arrays; 14 | 15 | import org.embulk.config.ConfigSource; 16 | import org.embulk.formatter.csv.CsvFormatterPlugin; 17 | import org.embulk.input.file.LocalFileInputPlugin; 18 | import org.embulk.output.MySQLOutputPlugin; 19 | import org.embulk.output.file.LocalFileOutputPlugin; 20 | import org.embulk.parser.csv.CsvParserPlugin; 21 | import org.embulk.spi.FileInputPlugin; 22 | import org.embulk.spi.FileOutputPlugin; 23 | import org.embulk.spi.FormatterPlugin; 24 | import org.embulk.spi.OutputPlugin; 25 | import org.embulk.spi.ParserPlugin; 26 | import org.embulk.test.EmbulkTests; 27 | import org.embulk.test.TestingEmbulk; 28 | import org.junit.Before; 29 | import org.junit.Rule; 30 | import org.junit.Test; 31 | 32 | public class CreateTableTest 33 | { 34 | private static final String BASIC_RESOURCE_PATH = "/org/embulk/output/mysql/test/expect/create_table/"; 35 | 36 | private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName) 37 | { 38 | return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName); 39 | } 40 | 41 | private static String readResource(String fileName) 42 | { 43 | return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName); 44 | } 45 | 46 | @Rule 47 | public TestingEmbulk embulk = TestingEmbulk.builder() 48 | .registerPlugin(FileInputPlugin.class, "file", LocalFileInputPlugin.class) 49 | .registerPlugin(ParserPlugin.class, "csv", CsvParserPlugin.class) 50 | .registerPlugin(FormatterPlugin.class, "csv", CsvFormatterPlugin.class) 51 | .registerPlugin(FileOutputPlugin.class, "file", LocalFileOutputPlugin.class) 52 | .registerPlugin(OutputPlugin.class, "mysql", MySQLOutputPlugin.class) 53 | .build(); 54 | 55 | private ConfigSource baseConfig; 56 | 57 | @Before 58 | public void setup() 59 | { 60 | baseConfig = MySQLTests.baseConfig(); 61 | execute(readResource("setup.sql")); // setup rows 62 | } 63 | 64 | @Test 65 | public void testTableOption() throws Exception 66 | { 67 | Path in1 = toPath("test1.csv"); 68 | TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_table_option.yml")), in1); 69 | assertThat(selectRecords(embulk, "test1"), is(readResource("test_table_option_expected.csv"))); 70 | //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff"))); 71 | } 72 | 73 | @Test 74 | public void testTableConstraint() throws Exception 75 | { 76 | Path in1 = toPath("test1.csv"); 77 | // exception will be thrown without table constraint because auto column must be defined as a key. 78 | TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_table_constraint.yml")), in1); 79 | assertThat(selectRecords(embulk, "test1"), is(readResource("test_table_constraint_expected.csv"))); 80 | //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff"))); 81 | } 82 | 83 | private Path toPath(String fileName) throws URISyntaxException 84 | { 85 | URL url = EmbulkTests.class.getResource(BASIC_RESOURCE_PATH + fileName); 86 | return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath()); 87 | } 88 | 89 | private String selectRecords(TestingEmbulk embulk, String tableName) 90 | { 91 | return MySQLTests.selectRecords(tableName, Arrays.asList("id", "value")); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/java/org/embulk/output/mysql/MySQLTests.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.mysql; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | import org.embulk.config.ConfigSource; 5 | import org.embulk.test.EmbulkTests; 6 | 7 | import java.sql.Connection; 8 | import java.sql.DriverManager; 9 | import java.sql.PreparedStatement; 10 | import java.sql.ResultSet; 11 | import java.sql.SQLException; 12 | import java.util.List; 13 | 14 | import static java.util.stream.Collectors.joining; 15 | 16 | public class MySQLTests 17 | { 18 | public static ConfigSource baseConfig() 19 | { 20 | return EmbulkTests.config("EMBULK_OUTPUT_MYSQL_TEST_CONFIG"); 21 | } 22 | 23 | public static Connection connect() throws SQLException 24 | { 25 | ConfigSource config = baseConfig(); 26 | 27 | String url = String.format("jdbc:mysql://%s:%s/%s?enabledTLSProtocols=TLSv1,TLSv1.1,TLSv1.2", 28 | config.get(String.class, "host"), 29 | config.get(String.class, "port", "3306"), 30 | config.get(String.class, "database")); 31 | return DriverManager.getConnection(url, config.get(String.class, "user"), config.get(String.class, "password")); 32 | } 33 | 34 | public static String executeQuery(String sql) 35 | { 36 | StringBuilder result = new StringBuilder(); 37 | try (Connection conn = connect(); 38 | PreparedStatement preparedStatement = conn.prepareStatement(sql); 39 | ResultSet resultSet = preparedStatement.executeQuery(); 40 | ) { 41 | while (resultSet.next()) { 42 | result.append(resultSet.getString(1)).append("\n"); 43 | } 44 | 45 | } catch (SQLException e) { 46 | throw new RuntimeException(e); 47 | } 48 | return result.toString(); 49 | } 50 | 51 | /** 52 | * execute sql content file 53 | * 54 | * @param sqlContent 55 | */ 56 | public static void execute(String sqlContent) 57 | { 58 | try (Connection conn = connect()) { 59 | for (String s : sqlContent.split(";")) { 60 | if (StringUtils.isBlank(s)) { 61 | continue; 62 | } 63 | try (PreparedStatement preparedStatement = conn.prepareStatement(s)) { 64 | preparedStatement.execute(); 65 | } 66 | } 67 | } catch (SQLException e) { 68 | throw new RuntimeException(e); 69 | } 70 | } 71 | 72 | public static String selectRecords(String tableName, List columnList) 73 | { 74 | final String cols = columnList.stream().collect((joining(","))); 75 | return executeQuery(String.format("SELECT concat(CONCAT_WS(',', %s)) from %s", cols, tableName)); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | 3 | create table test1 ( 4 | id char(4), 5 | int_item int, 6 | varchar_item varchar(8), 7 | primary key (id) 8 | ); 9 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test1.csv: -------------------------------------------------------------------------------- 1 | id:string,int_item:long,varchar_item:string 2 | A001,9,a 3 | A002,0,b 4 | A003,9,c 5 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_insert_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_insert_after_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,x 2 | A002,0,b 3 | A003,9,x 4 | B001,0,z 5 | B002,9,x 6 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_insert_direct_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert_direct 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_merge_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: merge 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_merge_after_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,x 2 | A002,0,b 3 | A003,9,x 4 | B001,0,z 5 | B002,9,x 6 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_merge_direct_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: merge_direct 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_replace_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: replace 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_replace_after_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,x 2 | A002,0,b 3 | A003,9,x 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_truncate_insert_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: truncate_insert 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_truncate_insert_after_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,x 2 | A002,0,b 3 | A003,9,x 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/basic/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | 3 | create table test1 ( 4 | id int, 5 | num decimal(12,2), 6 | str char(8), 7 | varstr varchar(8), 8 | dttm3 datetime(3), 9 | primary key(id) 10 | ); 11 | 12 | drop table if exists test_merge; 13 | create table test_merge ( 14 | id int, 15 | value1 varchar(8), 16 | value2 varchar(8), 17 | primary key(id) 18 | ); 19 | 20 | insert into test_merge values(11, 'A1', 'B1'); 21 | insert into test_merge values(12, 'A2', 'B2'); 22 | insert into test_merge values(13, 'A3', 'B3'); 23 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/basic/test1.csv: -------------------------------------------------------------------------------- 1 | id:long,num:string,str:string,varstr:string,dttm3:timestamp 2 | 1,123.40,test1,TEST1,2015-04-24 01:02:03.123 +0900 3 | 2,1234567890.12,test9999,TEST9999,2015-12-31 23:59:59.999 +0900 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/basic/test_invalid_time_zone.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert 3 | default_timezone: "Somewhere/Some_City" 4 | column_options: 5 | id: {type: 'int auto_increment primary key'} 6 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/basic/test_merge.csv: -------------------------------------------------------------------------------- 1 | id:long,value1:string,value2:string 2 | 12,a2,b2 3 | 14,a4,b4 4 | 16,a6,b6 5 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/basic/test_merge.yml: -------------------------------------------------------------------------------- 1 | table: test_merge 2 | mode: merge 3 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/basic/test_merge_expected.csv: -------------------------------------------------------------------------------- 1 | 11,A1,B1 2 | 12,a2,b2 3 | 13,A3,B3 4 | 14,a4,b4 5 | 16,a6,b6 6 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/basic/test_replace.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: replace 3 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/basic/test_replace_expected.csv: -------------------------------------------------------------------------------- 1 | 1,123.40,test1,TEST1,2015-04-23 16:02:03 2 | 2,1234567890.12,test9999,TEST9999,2015-12-31 15:00:00 3 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | 3 | create table test1 ( 4 | id char(4), 5 | int_item int, 6 | varchar_item varchar(8), 7 | primary key (id) 8 | ); 9 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test1.csv: -------------------------------------------------------------------------------- 1 | id:string,int_item:long,varchar_item:string 2 | A001,9,a 3 | A002,0,b 4 | A003,9,c 5 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_insert_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert 3 | before_load: "delete from test1 where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_insert_before_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,a 2 | A002,0,b 3 | A003,9,c 4 | B001,0,z 5 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_insert_direct_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert_direct 3 | before_load: "delete from test1 where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_merge_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: merge 3 | before_load: "delete from test1 where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_merge_before_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,a 2 | A002,0,b 3 | A003,9,c 4 | B001,0,z 5 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_merge_direct_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: merge_direct 3 | before_load: "delete from test1 where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_truncate_insert_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: truncate_insert 3 | before_load: "insert into test1 values('C001', 0, 'x')" 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/before_load/test_truncate_insert_before_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,a 2 | A002,0,b 3 | A003,9,c 4 | C001,0,x 5 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/create_table/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/create_table/test1.csv: -------------------------------------------------------------------------------- 1 | id:long,value:string 2 | 0,a 3 | 0,b 4 | 0,c 5 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/create_table/test_table_constraint.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert 3 | create_table_constraint: 'primary key(id)' 4 | column_options: 5 | id: {type: 'int auto_increment'} 6 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/create_table/test_table_constraint_expected.csv: -------------------------------------------------------------------------------- 1 | 1,a 2 | 2,b 3 | 3,c 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/create_table/test_table_option.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert 3 | create_table_option: 'auto_increment=100' 4 | column_options: 5 | id: {type: 'int auto_increment primary key'} 6 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/create_table/test_table_option_expected.csv: -------------------------------------------------------------------------------- 1 | 100,a 2 | 101,b 3 | 102,c 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/retry/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | 3 | create table test1 ( 4 | id char(8), 5 | num int, 6 | primary key (id) 7 | ); 8 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1.csv: -------------------------------------------------------------------------------- 1 | id:string,num:long 2 | A001,11 3 | A002,22 4 | A003,33 5 | A004,44 6 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert_direct 3 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_expected.csv: -------------------------------------------------------------------------------- 1 | A001,11 2 | A002,22 3 | A003,33 4 | A004,44 5 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_flushed_multiple_times.csv: -------------------------------------------------------------------------------- 1 | id:string,num:long 2 | A100,100 3 | A101,101 4 | A102,102 5 | A103,103 6 | A104,104 7 | A105,105 8 | A106,106 9 | A107,107 10 | A108,108 11 | A109,109 12 | A110,110 13 | A111,111 14 | A112,112 15 | A113,113 16 | A114,114 17 | A115,115 18 | A116,116 19 | A117,117 20 | A118,118 21 | A119,119 22 | A120,120 23 | A121,121 24 | A122,122 25 | A123,123 26 | A124,124 27 | A125,125 28 | A126,126 29 | A127,127 30 | A128,128 31 | A129,129 32 | A130,130 33 | A131,131 34 | A132,132 35 | A133,133 36 | A134,134 37 | A135,135 38 | A136,136 39 | A137,137 40 | A138,138 41 | A139,139 42 | A140,140 43 | A141,141 44 | A142,142 45 | A143,143 46 | A144,144 47 | A145,145 48 | A146,146 49 | A147,147 50 | A148,148 51 | A149,149 52 | A150,150 53 | A151,151 54 | A152,152 55 | A153,153 56 | A154,154 57 | A155,155 58 | A156,156 59 | A157,157 60 | A158,158 61 | A159,159 62 | A160,160 63 | A161,161 64 | A162,162 65 | A163,163 66 | A164,164 67 | A165,165 68 | A166,166 69 | A167,167 70 | A168,168 71 | A169,169 72 | A170,170 73 | A171,171 74 | A172,172 75 | A173,173 76 | A174,174 77 | A175,175 78 | A176,176 79 | A177,177 80 | A178,178 81 | A179,179 82 | A180,180 83 | A181,181 84 | A182,182 85 | A183,183 86 | A184,184 87 | A185,185 88 | A186,186 89 | A187,187 90 | A188,188 91 | A189,189 92 | A190,190 93 | A191,191 94 | A192,192 95 | A193,193 96 | A194,194 97 | A195,195 98 | A196,196 99 | A197,197 100 | A198,198 101 | A199,199 102 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_flushed_multiple_times.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert_direct 3 | batch_size: 1000 4 | -------------------------------------------------------------------------------- /embulk-output-mysql/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_flushed_multiple_times_expected.csv: -------------------------------------------------------------------------------- 1 | A100,100 2 | A101,101 3 | A102,102 4 | A103,103 5 | A104,104 6 | A105,105 7 | A106,106 8 | A107,107 9 | A108,108 10 | A109,109 11 | A110,110 12 | A111,111 13 | A112,112 14 | A113,113 15 | A114,114 16 | A115,115 17 | A116,116 18 | A117,117 19 | A118,118 20 | A119,119 21 | A120,120 22 | A121,121 23 | A122,122 24 | A123,123 25 | A124,124 26 | A125,125 27 | A126,126 28 | A127,127 29 | A128,128 30 | A129,129 31 | A130,130 32 | A131,131 33 | A132,132 34 | A133,133 35 | A134,134 36 | A135,135 37 | A136,136 38 | A137,137 39 | A138,138 40 | A139,139 41 | A140,140 42 | A141,141 43 | A142,142 44 | A143,143 45 | A144,144 46 | A145,145 47 | A146,146 48 | A147,147 49 | A148,148 50 | A149,149 51 | A150,150 52 | A151,151 53 | A152,152 54 | A153,153 55 | A154,154 56 | A155,155 57 | A156,156 58 | A157,157 59 | A158,158 60 | A159,159 61 | A160,160 62 | A161,161 63 | A162,162 64 | A163,163 65 | A164,164 66 | A165,165 67 | A166,166 68 | A167,167 69 | A168,168 70 | A169,169 71 | A170,170 72 | A171,171 73 | A172,172 74 | A173,173 75 | A174,174 76 | A175,175 77 | A176,176 78 | A177,177 79 | A178,178 80 | A179,179 81 | A180,180 82 | A181,181 83 | A182,182 84 | A183,183 85 | A184,184 86 | A185,185 87 | A186,186 88 | A187,187 89 | A188,188 90 | A189,189 91 | A190,190 92 | A191,191 93 | A192,192 94 | A193,193 95 | A194,194 96 | A195,195 97 | A196,196 98 | A197,197 99 | A198,198 100 | A199,199 101 | -------------------------------------------------------------------------------- /embulk-output-postgresql/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation(project(path: ":embulk-output-jdbc", configuration: "runtimeElements")) 3 | 4 | compileOnly "org.postgresql:postgresql:9.4-1205-jdbc41" 5 | defaultJdbcDriver 'org.postgresql:postgresql:9.4-1205-jdbc41' 6 | 7 | testImplementation project(':embulk-output-jdbc').sourceSets.test.output 8 | testImplementation "org.postgresql:postgresql:9.4-1205-jdbc41" 9 | 10 | testImplementation "org.embulk:embulk-formatter-csv:0.10.42" 11 | testImplementation "org.embulk:embulk-input-file:0.10.42" 12 | testImplementation "org.embulk:embulk-output-file:0.10.42" 13 | testImplementation "org.embulk:embulk-parser-csv:0.10.42" 14 | } 15 | 16 | embulkPlugin { 17 | mainClass = "org.embulk.output.PostgreSQLOutputPlugin" 18 | category = "output" 19 | type = "postgresql" 20 | } 21 | 22 | publishing { 23 | publications { 24 | maven(MavenPublication) { 25 | pom { // https://central.sonatype.org/pages/requirements.html 26 | developers { 27 | developer { 28 | name = "Sadayuki Furuhashi" 29 | email = "frsyuki@gmail.com" 30 | } 31 | developer { 32 | name = "takumakanari" 33 | email = "chemtrails.t@gmail.com" 34 | } 35 | developer { 36 | name = "Hitoshi Tanaka" 37 | email = "thitoshi@cac.co.jp" 38 | } 39 | developer { 40 | name = "Hiroyuki Sato" 41 | email = "hiroysato@gmail.com" 42 | } 43 | developer { 44 | name = "Dai MIKURUBE" 45 | email = "dmikurube@treasure-data.com" 46 | } 47 | } 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /embulk-output-postgresql/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 9 | org.embulk:embulk-spi:0.10.49=compileClasspath 10 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 11 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 12 | org.embulk:embulk-util-retryhelper:0.8.2=compileClasspath,runtimeClasspath 13 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 14 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 15 | org.msgpack:msgpack-core:0.8.24=compileClasspath 16 | org.postgresql:postgresql:9.4-1205-jdbc41=compileClasspath 17 | org.slf4j:slf4j-api:2.0.7=compileClasspath 18 | empty= 19 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.postgresql; 2 | 3 | import java.io.File; 4 | import java.io.FileInputStream; 5 | import java.io.IOException; 6 | import java.sql.SQLException; 7 | 8 | import org.embulk.output.jdbc.JdbcOutputConnector; 9 | import org.embulk.output.jdbc.JdbcSchema; 10 | import org.embulk.output.jdbc.TableIdentifier; 11 | import org.postgresql.copy.CopyManager; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | public class PostgreSQLCopyBatchInsert 16 | extends AbstractPostgreSQLCopyBatchInsert 17 | { 18 | private static final Logger logger = LoggerFactory.getLogger(PostgreSQLCopyBatchInsert.class); 19 | private final JdbcOutputConnector connector; 20 | 21 | private PostgreSQLOutputConnection connection = null; 22 | private CopyManager copyManager = null; 23 | private String copySql = null; 24 | private long totalRows; 25 | 26 | public PostgreSQLCopyBatchInsert(JdbcOutputConnector connector) throws IOException, SQLException 27 | { 28 | super(); 29 | this.connector = connector; 30 | } 31 | 32 | @Override 33 | public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException 34 | { 35 | this.connection = (PostgreSQLOutputConnection)connector.connect(true); 36 | this.copySql = connection.buildCopySql(loadTable, insertSchema); 37 | this.copyManager = connection.newCopyManager(); 38 | logger.info("Copy SQL: "+copySql); 39 | } 40 | 41 | @Override 42 | public void flush() throws IOException, SQLException 43 | { 44 | File file = closeCurrentFile(); // flush buffered data in writer 45 | if (getBatchWeight() == 0) return; 46 | 47 | logger.info(String.format("Loading %,d rows (%,d bytes)", batchRows, file.length())); 48 | long startTime = System.currentTimeMillis(); 49 | FileInputStream in = new FileInputStream(file); 50 | try { 51 | // TODO check age of connection and call isValid if it's old and reconnect if it's invalid 52 | copyManager.copyIn(copySql, in); 53 | } finally { 54 | in.close(); 55 | } 56 | double seconds = (System.currentTimeMillis() - startTime) / 1000.0; 57 | 58 | totalRows += batchRows; 59 | batchRows = 0; 60 | logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows)); 61 | 62 | openNewFile(); 63 | file.delete(); 64 | } 65 | 66 | @Override 67 | public void finish() throws IOException ,SQLException 68 | { 69 | } 70 | 71 | @Override 72 | public void close() throws IOException, SQLException 73 | { 74 | closeCurrentFile().delete(); 75 | if (connection != null) { 76 | connection.close(); 77 | connection = null; 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.postgresql; 2 | 3 | import java.util.Properties; 4 | import java.sql.Connection; 5 | import java.sql.DriverManager; 6 | import java.sql.SQLException; 7 | import java.util.Optional; 8 | 9 | import org.embulk.output.jdbc.JdbcOutputConnection; 10 | import org.embulk.output.jdbc.AbstractJdbcOutputConnector; 11 | import org.embulk.output.jdbc.TransactionIsolation; 12 | 13 | public class PostgreSQLOutputConnector 14 | extends AbstractJdbcOutputConnector 15 | { 16 | private final String url; 17 | private final Properties properties; 18 | private final String schemaName; 19 | private final String roleName; 20 | 21 | public PostgreSQLOutputConnector(String url, Properties properties, String schemaName, 22 | Optional transactionIsolation, String roleName) 23 | { 24 | super(transactionIsolation); 25 | this.url = url; 26 | this.properties = properties; 27 | this.schemaName = schemaName; 28 | this.roleName = roleName; 29 | } 30 | 31 | @Override 32 | protected JdbcOutputConnection connect() throws SQLException 33 | { 34 | Connection c = DriverManager.getConnection(url, properties); 35 | try { 36 | PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, roleName); 37 | c = null; 38 | return con; 39 | } finally { 40 | if (c != null) { 41 | c.close(); 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/main/java/org/embulk/output/postgresql/setter/PostgreSQLColumnSetterFactory.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.postgresql.setter; 2 | 3 | import java.time.ZoneId; 4 | import org.embulk.output.jdbc.BatchInsert; 5 | import org.embulk.output.jdbc.JdbcColumn; 6 | import org.embulk.output.jdbc.JdbcColumnOption; 7 | import org.embulk.output.jdbc.setter.ColumnSetter; 8 | import org.embulk.output.jdbc.setter.ColumnSetterFactory; 9 | import org.embulk.output.jdbc.setter.JsonColumnSetter; 10 | 11 | public class PostgreSQLColumnSetterFactory 12 | extends ColumnSetterFactory 13 | { 14 | public PostgreSQLColumnSetterFactory(final BatchInsert batch, final ZoneId defaultTimeZone) 15 | { 16 | super(batch, defaultTimeZone); 17 | } 18 | 19 | @Override 20 | public ColumnSetter newCoalesceColumnSetter(JdbcColumn column, JdbcColumnOption option) 21 | { 22 | if (column.getSimpleTypeName().equalsIgnoreCase("json") || column.getSimpleTypeName().equalsIgnoreCase("jsonb")) { 23 | // actually "JSON"/"JSONB" 24 | return new JsonColumnSetter(batch, column, newDefaultValueSetter(column, option)); 25 | } else { 26 | return super.newCoalesceColumnSetter(column, option); 27 | } 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/java/org/embulk/output/postgresql/PostgreSQLTests.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.postgresql; 2 | 3 | import org.embulk.test.EmbulkTests; 4 | import org.embulk.test.TestingEmbulk; 5 | 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.io.OutputStream; 9 | import java.nio.file.Files; 10 | import java.nio.file.Path; 11 | import java.util.List; 12 | import java.util.stream.Collectors; 13 | 14 | import org.embulk.config.ConfigSource; 15 | 16 | import static java.util.Locale.ENGLISH; 17 | import static java.util.stream.Collectors.joining; 18 | import static org.embulk.test.EmbulkTests.readSortedFile; 19 | 20 | public class PostgreSQLTests 21 | { 22 | public static ConfigSource baseConfig() 23 | { 24 | return EmbulkTests.config("EMBULK_OUTPUT_POSTGRESQL_TEST_CONFIG"); 25 | } 26 | 27 | public static void execute(String sql) 28 | { 29 | System.out.println(sql); 30 | ConfigSource config = baseConfig(); 31 | ProcessBuilder pb = new ProcessBuilder( 32 | "psql", "-w", 33 | "--set", "ON_ERROR_STOP=1", 34 | "--host", config.get(String.class, "host"), 35 | "--username", config.get(String.class, "user"), 36 | "--dbname", config.get(String.class, "database"), 37 | "-c", sql); 38 | pb.environment().put("PGPASSWORD", config.get(String.class, "password")); 39 | pb.redirectErrorStream(true); 40 | int code; 41 | try { 42 | Process process = pb.start(); 43 | InputStream inputStream = process.getInputStream(); 44 | byte[] buffer = new byte[8192]; 45 | int readSize; 46 | while ((readSize = inputStream.read(buffer)) != -1) { 47 | System.out.write(buffer, 0, readSize); 48 | } 49 | code = process.waitFor(); 50 | } catch (IOException | InterruptedException ex) { 51 | throw new RuntimeException(ex); 52 | } 53 | if (code != 0) { 54 | throw new RuntimeException(String.format(ENGLISH, 55 | "Command finished with non-zero exit code. Exit code is %d.", code)); 56 | } 57 | } 58 | 59 | public static String selectRecords(TestingEmbulk embulk, String tableName) throws IOException 60 | { 61 | Path temp = embulk.createTempFile("txt"); 62 | Files.delete(temp); 63 | execute("\\copy " + tableName + " to '" + temp.toString().replace("\\", "\\\\") + "' delimiter ','"); 64 | return readSortedFile(temp); 65 | } 66 | 67 | public static String selectRecords(TestingEmbulk embulk, String tableName, List columnList) throws IOException 68 | { 69 | Path temp = embulk.createTempFile("txt"); 70 | Files.delete(temp); 71 | final String cols = columnList.stream().collect(Collectors.joining(",")); 72 | execute(String.format("\\COPY (SELECT %s FROM %s) TO '%s' With CSV DELIMITER ',';", cols, tableName, temp.toString().replace("\\", "\\\\"))); 73 | return readSortedFile(temp); 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | 3 | create table test1 ( 4 | id char(4), 5 | int_item int, 6 | varchar_item varchar(8), 7 | primary key (id) 8 | ); 9 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test1.csv: -------------------------------------------------------------------------------- 1 | id:string,int_item:long,varchar_item:string 2 | A001,9,a 3 | A002,0,b 4 | A003,9,c 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_insert_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_insert_after_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,x 2 | A002,0,b 3 | A003,9,x 4 | B001,0,z 5 | B002,9,x 6 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_insert_direct_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert_direct 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_merge_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: merge 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_merge_after_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,x 2 | A002,0,b 3 | A003,9,x 4 | B001,0,z 5 | B002,9,x 6 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_merge_direct_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: merge_direct 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_replace_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: replace 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_replace_after_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,x 2 | A002,0,b 3 | A003,9,x 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_truncate_insert_after_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: truncate_insert 3 | after_load: "update test1 set varchar_item = 'x' where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/after_load/test_truncate_insert_after_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,x 2 | A002,0,b 3 | A003,9,x 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test_number; 2 | 3 | create table test_number ( 4 | id char(4), 5 | bool_item bool, 6 | smallint_item smallint, 7 | int_item int, 8 | bigint_item bigint, 9 | real_item real, 10 | double_item double precision, 11 | money_item money, 12 | numeric_item numeric(8,3), 13 | primary key (id) 14 | ); 15 | 16 | drop table if exists test_string; 17 | 18 | create table test_string ( 19 | id int, 20 | char_item char(4), 21 | varchar_item varchar(4), 22 | text_item text, 23 | primary key (id) 24 | ); 25 | 26 | drop table if exists test_timestamp; 27 | 28 | create table test_timestamp ( 29 | id int, 30 | date_item date, 31 | time_item time, 32 | timestamp_item timestamp, 33 | primary key (id) 34 | ); 35 | 36 | drop table if exists test_json; 37 | 38 | create table test_json ( 39 | id int, 40 | json_item json, 41 | jsonb_item jsonb, 42 | primary key (id) 43 | ); 44 | 45 | drop table if exists test_merge; 46 | 47 | create table test_merge ( 48 | id int, 49 | value1 varchar(8), 50 | value2 varchar(8), 51 | primary key(id) 52 | ); 53 | 54 | insert into test_merge values(11, 'A1', 'B1'); 55 | insert into test_merge values(12, 'A2', 'B2'); 56 | insert into test_merge values(13, 'A3', 'B3'); 57 | 58 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json.csv: -------------------------------------------------------------------------------- 1 | id:long,json_item:json,jsonb_item:json 2 | 1,null,null 3 | 2,88,99 4 | 3,{"x":"@"},{"y":{"z":[5]}} 5 | 4,, 6 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json.yml: -------------------------------------------------------------------------------- 1 | table: test_json 2 | mode: insert 3 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_json_expected.csv: -------------------------------------------------------------------------------- 1 | 1,null,null 2 | 2,88,99 3 | 3,{"x":"@"},{"y": {"z": [5]}} 4 | 4,\N,\N 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_merge.csv: -------------------------------------------------------------------------------- 1 | id:long,value1:string,value2:string 2 | 12,a2,b2 3 | 14,a4,b4 4 | 16,a6,b6 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_merge.yml: -------------------------------------------------------------------------------- 1 | table: test_merge 2 | mode: merge 3 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_merge_expected.csv: -------------------------------------------------------------------------------- 1 | 11,A1,B1 2 | 12,a2,b2 3 | 13,A3,B3 4 | 14,a4,b4 5 | 16,a6,b6 6 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_merge_keys.csv: -------------------------------------------------------------------------------- 1 | id:long,value1:string,value2:string 2 | 22,A2,b2 3 | 24,A4,b4 4 | 26,A6,b6 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_merge_keys.yml: -------------------------------------------------------------------------------- 1 | table: test_merge 2 | mode: merge 3 | merge_keys: 4 | - value1 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_merge_keys_expected.csv: -------------------------------------------------------------------------------- 1 | 11,A1,B1 2 | 13,A3,B3 3 | 22,A2,b2 4 | 24,A4,b4 5 | 26,A6,b6 6 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_merge_rule.yml: -------------------------------------------------------------------------------- 1 | table: test_merge 2 | mode: merge 3 | merge_rule: ["value1 = S.value1", "value2 = test_merge.value2 || S.value2"] 4 | 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_merge_rule_expected.csv: -------------------------------------------------------------------------------- 1 | 11,A1,B1 2 | 12,a2,B2b2 3 | 13,A3,B3 4 | 14,a4,b4 5 | 16,a6,b6 6 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number.csv: -------------------------------------------------------------------------------- 1 | id:string,bool_item:long,smallint_item:long,int_item:long,bigint_item:long,real_item:double,double_item:double,money_item:string,numeric_item:string 2 | A001,0,0,0,0,0,0,0,0 3 | A002,1,12345,123456789,123456789012,123.45,123456.789,1234,1234.567 4 | A003,0,-1,-1,-1,-1,-1,-1,-1 5 | A004,,,,,,,, 6 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number.yml: -------------------------------------------------------------------------------- 1 | table: test_number 2 | mode: insert 3 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_number_expected.csv: -------------------------------------------------------------------------------- 1 | A001,f,0,0,0,0,0,$0.00,0.000 2 | A002,t,12345,123456789,123456789012,123.45,123456.789,$1\,234.00,1234.567 3 | A003,f,-1,-1,-1,-1,-1,-$1.00,-1.000 4 | A004,\N,\N,\N,\N,\N,\N,\N,\N 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_replace.yml: -------------------------------------------------------------------------------- 1 | table: test_string 2 | mode: replace 3 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_replace_expected.csv: -------------------------------------------------------------------------------- 1 | 1,a,b,c 2 | 2,abc,def,ABCDEFGHIJ 3 | 3,\N,\N,\N 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string.csv: -------------------------------------------------------------------------------- 1 | id:long,char_item:string,varchar_item:string,text_item:string 2 | 1,a,b,c 3 | 2,abc,def,ABCDEFGHIJ 4 | 3,,, 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string.yml: -------------------------------------------------------------------------------- 1 | table: test_string 2 | mode: insert 3 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_string_expected.csv: -------------------------------------------------------------------------------- 1 | 1,a ,b,c 2 | 2,abc ,def,ABCDEFGHIJ 3 | 3,\N,\N,\N 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp.csv: -------------------------------------------------------------------------------- 1 | id:long,date_item:timestamp,time_item:timestamp,timestamp_item:timestamp 2 | 1,2019-1-2 00:00:00.000 +0900,2001-1-1 12:34:56.000 +0900,2019-12-31 23:59:59.000 +0900 3 | 2,,, 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp.yml: -------------------------------------------------------------------------------- 1 | table: test_timestamp 2 | mode: insert 3 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/basic/test_timestamp_expected.csv: -------------------------------------------------------------------------------- 1 | 1,2019-01-01,03:34:56,2019-12-31 14:59:59 2 | 2,\N,\N,\N 3 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/setup.sql: -------------------------------------------------------------------------------- 1 | drop table if exists test1; 2 | 3 | create table test1 ( 4 | id char(4), 5 | int_item int, 6 | varchar_item varchar(8), 7 | primary key (id) 8 | ); 9 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test1.csv: -------------------------------------------------------------------------------- 1 | id:string,int_item:long,varchar_item:string 2 | A001,9,a 3 | A002,0,b 4 | A003,9,c 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_insert_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert 3 | before_load: "delete from test1 where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_insert_before_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,a 2 | A002,0,b 3 | A003,9,c 4 | B001,0,z 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_insert_direct_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: insert_direct 3 | before_load: "delete from test1 where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_merge_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: merge 3 | before_load: "delete from test1 where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_merge_before_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,a 2 | A002,0,b 3 | A003,9,c 4 | B001,0,z 5 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_merge_direct_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: merge_direct 3 | before_load: "delete from test1 where int_item = 9" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_truncate_insert_before_load.yml: -------------------------------------------------------------------------------- 1 | table: test1 2 | mode: truncate_insert 3 | before_load: "insert into test1 values('C001', 0, 'x')" 4 | -------------------------------------------------------------------------------- /embulk-output-postgresql/src/test/resources/org/embulk/output/postgresql/test/expect/before_load/test_truncate_insert_before_load_expected.csv: -------------------------------------------------------------------------------- 1 | A001,9,a 2 | A002,0,b 3 | A003,9,c 4 | C001,0,x 5 | -------------------------------------------------------------------------------- /embulk-output-redshift/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation(project(path: ":embulk-output-jdbc", configuration: "runtimeElements")) 3 | implementation(project(path: ":embulk-output-postgresql", configuration: "runtimeElements")) 4 | implementation "org.postgresql:postgresql:9.4-1205-jdbc41" 5 | 6 | implementation("com.amazonaws:aws-java-sdk-s3:1.11.523") { 7 | exclude group: "joda-time", module: "joda-time" 8 | exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations" 9 | exclude group: "com.fasterxml.jackson.core", module: "jackson-databind" 10 | exclude group: "com.fasterxml.jackson.core", module: "jackson-core" 11 | } 12 | implementation("com.amazonaws:aws-java-sdk-sts:1.11.523") { 13 | exclude group: "joda-time", module: "joda-time" 14 | exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations" 15 | exclude group: "com.fasterxml.jackson.core", module: "jackson-databind" 16 | exclude group: "com.fasterxml.jackson.core", module: "jackson-core" 17 | } 18 | implementation("org.embulk:embulk-util-aws-credentials:0.4.2") { 19 | exclude group: "org.slf4j", module: "slf4j-api" 20 | } 21 | 22 | // Joda-Time has been removed from Embulk since v0.10.30. 23 | // Ordinary plugins no longer need Joda-Time, but embulk-output-redshift depends on Joda-Time as aws-java-sdk-s3 depends. 24 | implementation "joda-time:joda-time:2.9.2" 25 | 26 | testImplementation project(':embulk-output-jdbc').sourceSets.test.output 27 | } 28 | 29 | embulkPlugin { 30 | mainClass = "org.embulk.output.RedshiftOutputPlugin" 31 | category = "output" 32 | type = "redshift" 33 | } 34 | 35 | publishing { 36 | publications { 37 | maven(MavenPublication) { 38 | pom { // https://central.sonatype.org/pages/requirements.html 39 | developers { 40 | developer { 41 | name = "Sadayuki Furuhashi" 42 | email = "frsyuki@gmail.com" 43 | } 44 | developer { 45 | name = "Hitoshi Tanaka" 46 | email = "thitoshi@cac.co.jp" 47 | } 48 | developer { 49 | name = "Hiroyuki Sato" 50 | email = "hiroysato@gmail.com" 51 | } 52 | developer { 53 | name = "Antoine Augusti" 54 | email = "hi@antoine-augusti.fr" 55 | } 56 | developer { 57 | name = "Michael Jalkio" 58 | email = "mjalkio@gmail.com" 59 | } 60 | developer { 61 | name = "Yutaka Nishimura" 62 | email = "ytk.nishimura@gmail.com" 63 | } 64 | developer { 65 | name = "Dai MIKURUBE" 66 | email = "dmikurube@treasure-data.com" 67 | } 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /embulk-output-redshift/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.amazonaws:aws-java-sdk-core:1.11.523=compileClasspath,runtimeClasspath 5 | com.amazonaws:aws-java-sdk-kms:1.11.523=compileClasspath,runtimeClasspath 6 | com.amazonaws:aws-java-sdk-s3:1.11.523=compileClasspath,runtimeClasspath 7 | com.amazonaws:aws-java-sdk-sts:1.11.523=compileClasspath,runtimeClasspath 8 | com.amazonaws:jmespath-java:1.11.523=compileClasspath,runtimeClasspath 9 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 10 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 11 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 12 | com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.6.7=compileClasspath,runtimeClasspath 13 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 14 | commons-codec:commons-codec:1.10=compileClasspath,runtimeClasspath 15 | commons-logging:commons-logging:1.2=compileClasspath,runtimeClasspath 16 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 17 | joda-time:joda-time:2.9.2=compileClasspath,runtimeClasspath 18 | org.apache.httpcomponents:httpclient:4.5.5=compileClasspath,runtimeClasspath 19 | org.apache.httpcomponents:httpcore:4.4.9=compileClasspath,runtimeClasspath 20 | org.embulk:embulk-spi:0.10.49=compileClasspath 21 | org.embulk:embulk-util-aws-credentials:0.4.2=compileClasspath,runtimeClasspath 22 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 23 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 24 | org.embulk:embulk-util-retryhelper:0.8.2=compileClasspath,runtimeClasspath 25 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 26 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 27 | org.msgpack:msgpack-core:0.8.24=compileClasspath 28 | org.postgresql:postgresql:9.4-1205-jdbc41=compileClasspath,runtimeClasspath 29 | org.slf4j:jcl-over-slf4j:1.7.36=compileClasspath,runtimeClasspath 30 | org.slf4j:slf4j-api:2.0.7=compileClasspath 31 | software.amazon.ion:ion-java:1.0.2=compileClasspath,runtimeClasspath 32 | empty= 33 | -------------------------------------------------------------------------------- /embulk-output-redshift/src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.redshift; 2 | 3 | import java.util.Properties; 4 | import java.sql.Driver; 5 | import java.sql.Connection; 6 | import java.sql.SQLException; 7 | import java.util.Optional; 8 | 9 | import org.embulk.output.jdbc.AbstractJdbcOutputConnector; 10 | import org.embulk.output.jdbc.JdbcOutputConnection; 11 | import org.embulk.output.jdbc.TransactionIsolation; 12 | 13 | public class RedshiftOutputConnector 14 | extends AbstractJdbcOutputConnector 15 | { 16 | private static final Driver driver = new org.postgresql.Driver(); 17 | 18 | private final String url; 19 | private final Properties properties; 20 | private final String schemaName; 21 | 22 | public RedshiftOutputConnector(String url, Properties properties, String schemaName, 23 | Optional transactionIsolation) 24 | { 25 | super(transactionIsolation); 26 | this.url = url; 27 | this.properties = properties; 28 | this.schemaName = schemaName; 29 | } 30 | 31 | @Override 32 | protected JdbcOutputConnection connect() throws SQLException 33 | { 34 | Connection c = driver.connect(url, properties); 35 | try { 36 | RedshiftOutputConnection con = new RedshiftOutputConnection(c, schemaName); 37 | c = null; 38 | return con; 39 | } finally { 40 | if (c != null) { 41 | c.close(); 42 | } 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | implementation(project(path: ":embulk-output-jdbc", configuration: "runtimeElements")) 3 | implementation 'com.microsoft.sqlserver:mssql-jdbc:7.2.2.jre8' 4 | implementation 'net.sourceforge.jtds:jtds:1.3.1' 5 | 6 | // embulk-output-oracle has used jnr-ffi included in JRuby, which had been embedded with Embulk till v0.10.20. 7 | // JRuby is, however, no longer loaded in Embulk's top-level class loader, nor built-in in Embulk since v0.10.21. 8 | // 9 | // Here, it is including dependencies on jnr-ffi and its transitive dependencies, instead of JRuby embedded in Embulk. 10 | // The versions are the same as JRuby 9.1.15.0, which were embedded in Embulk v0.10.20 and earlier for a long time. 11 | implementation "com.github.jnr:jnr-ffi:2.1.7" 12 | 13 | implementation "com.github.jnr:jnr-x86asm:1.0.2" 14 | implementation "com.github.jnr:jffi:1.2.16" 15 | implementation "org.ow2.asm:asm-analysis:5.0.4" 16 | implementation "org.ow2.asm:asm-commons:5.0.4" 17 | implementation "org.ow2.asm:asm-tree:5.0.4" 18 | implementation "org.ow2.asm:asm-util:5.0.4" 19 | implementation "org.ow2.asm:asm:5.0.4" 20 | 21 | testImplementation project(':embulk-output-jdbc').sourceSets.test.output 22 | } 23 | 24 | embulkPlugin { 25 | mainClass = "org.embulk.output.SQLServerOutputPlugin" 26 | category = "output" 27 | type = "sqlserver" 28 | } 29 | 30 | publishing { 31 | publications { 32 | maven(MavenPublication) { 33 | pom { // https://central.sonatype.org/pages/requirements.html 34 | developers { 35 | developer { 36 | name = "Hitoshi Tanaka" 37 | email = "thitoshi@cac.co.jp" 38 | } 39 | developer { 40 | name = "Yui Kitsu" 41 | email = "kitsuyui@kitsuyui.com" 42 | } 43 | developer { 44 | name = "Hieu Duong" 45 | email = "duongminhhieu89@gmail.com" 46 | } 47 | developer { 48 | name = "Dai MIKURUBE" 49 | email = "dmikurube@treasure-data.com" 50 | } 51 | } 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/gradle.lockfile: -------------------------------------------------------------------------------- 1 | # This is a Gradle generated file for dependency locking. 2 | # Manual edits can break the build and are not advised. 3 | # This file is expected to be part of source control. 4 | com.fasterxml.jackson.core:jackson-annotations:2.6.7=compileClasspath,runtimeClasspath 5 | com.fasterxml.jackson.core:jackson-core:2.6.7=compileClasspath,runtimeClasspath 6 | com.fasterxml.jackson.core:jackson-databind:2.6.7.5=compileClasspath,runtimeClasspath 7 | com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7=compileClasspath,runtimeClasspath 8 | com.github.jnr:jffi:1.2.16=compileClasspath,runtimeClasspath 9 | com.github.jnr:jnr-ffi:2.1.7=compileClasspath,runtimeClasspath 10 | com.github.jnr:jnr-x86asm:1.0.2=compileClasspath,runtimeClasspath 11 | com.microsoft.sqlserver:mssql-jdbc:7.2.2.jre8=compileClasspath,runtimeClasspath 12 | javax.validation:validation-api:1.1.0.Final=compileClasspath,runtimeClasspath 13 | net.sourceforge.jtds:jtds:1.3.1=compileClasspath,runtimeClasspath 14 | org.embulk:embulk-spi:0.10.49=compileClasspath 15 | org.embulk:embulk-util-config:0.3.4=compileClasspath,runtimeClasspath 16 | org.embulk:embulk-util-json:0.3.0=compileClasspath,runtimeClasspath 17 | org.embulk:embulk-util-retryhelper:0.8.2=compileClasspath,runtimeClasspath 18 | org.embulk:embulk-util-rubytime:0.3.3=compileClasspath,runtimeClasspath 19 | org.embulk:embulk-util-timestamp:0.2.2=compileClasspath,runtimeClasspath 20 | org.msgpack:msgpack-core:0.8.24=compileClasspath 21 | org.ow2.asm:asm-analysis:5.0.4=compileClasspath,runtimeClasspath 22 | org.ow2.asm:asm-commons:5.0.4=compileClasspath,runtimeClasspath 23 | org.ow2.asm:asm-tree:5.0.4=compileClasspath,runtimeClasspath 24 | org.ow2.asm:asm-util:5.0.4=compileClasspath,runtimeClasspath 25 | org.ow2.asm:asm:5.0.4=compileClasspath,runtimeClasspath 26 | org.slf4j:slf4j-api:2.0.7=compileClasspath 27 | empty= 28 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/main/java/org/embulk/output/sqlserver/InsertMethod.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.sqlserver; 2 | 3 | import java.util.Locale; 4 | 5 | import org.embulk.config.ConfigException; 6 | 7 | import com.fasterxml.jackson.annotation.JsonCreator; 8 | import com.fasterxml.jackson.annotation.JsonValue; 9 | 10 | public enum InsertMethod 11 | { 12 | NORMAL, 13 | NATIVE; 14 | 15 | @JsonValue 16 | @Override 17 | public String toString() 18 | { 19 | return name().toLowerCase(Locale.ENGLISH); 20 | } 21 | 22 | @JsonCreator 23 | public static InsertMethod fromString(String value) 24 | { 25 | for (InsertMethod insertMethod : InsertMethod.values()) { 26 | if (insertMethod.toString().equals(value)) { 27 | return insertMethod; 28 | } 29 | } 30 | throw new ConfigException(String.format("Unknown insert_method '%s'.", value)); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/main/java/org/embulk/output/sqlserver/SQLServerOutputConnector.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.sqlserver; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.PreparedStatement; 6 | import java.sql.ResultSet; 7 | import java.sql.SQLException; 8 | import java.util.Properties; 9 | import java.util.Optional; 10 | 11 | import org.embulk.output.jdbc.JdbcOutputConnection; 12 | import org.embulk.output.jdbc.AbstractJdbcOutputConnector; 13 | import org.embulk.output.jdbc.TransactionIsolation; 14 | import org.slf4j.Logger; 15 | import org.slf4j.LoggerFactory; 16 | 17 | public class SQLServerOutputConnector 18 | extends AbstractJdbcOutputConnector 19 | { 20 | private static final Logger logger = LoggerFactory.getLogger(SQLServerOutputConnector.class); 21 | 22 | private final String url; 23 | private final Properties properties; 24 | private final String schemaName; 25 | 26 | public SQLServerOutputConnector(String url, Properties properties, String schemaName, 27 | Optional transactionIsolation) 28 | { 29 | super(transactionIsolation); 30 | this.url = url; 31 | this.properties = properties; 32 | this.schemaName = schemaName; 33 | } 34 | 35 | @Override 36 | protected JdbcOutputConnection connect() throws SQLException 37 | { 38 | Connection c = DriverManager.getConnection(url, properties); 39 | if (c == null) { 40 | // driver.connect returns null when url is "jdbc:mysql://...". 41 | throw new SQLException("Invalid url : " + url); 42 | } 43 | 44 | String schemaName = this.schemaName; 45 | if (schemaName == null) { 46 | // get default schema name for user (Connection#getSchema won't work) 47 | try { 48 | try (PreparedStatement statement = c.prepareStatement("SELECT default_schema_name FROM sys.database_principals WHERE name = ?")) { 49 | statement.setString(1, properties.getProperty("user")); 50 | try (ResultSet rs = statement.executeQuery()) { 51 | if (rs.next()) { 52 | schemaName = rs.getString(1); 53 | } 54 | } 55 | } 56 | } catch (SQLException e) { 57 | logger.warn("Cannot specify default schema : " + e); 58 | } 59 | } 60 | 61 | try { 62 | SQLServerOutputConnection con = new SQLServerOutputConnection(c, schemaName); 63 | c = null; 64 | return con; 65 | 66 | } finally { 67 | if (c != null) { 68 | c.close(); 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/main/java/org/embulk/output/sqlserver/SmallDateTimeFormat.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.sqlserver; 2 | 3 | import java.text.FieldPosition; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | 7 | 8 | public class SmallDateTimeFormat extends SimpleDateFormat 9 | { 10 | public SmallDateTimeFormat(String pattern) 11 | { 12 | super(pattern); 13 | } 14 | 15 | @Override 16 | public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition pos) 17 | { 18 | long time = date.getTime(); 19 | 20 | // round seconds 21 | long underMinutes = time % 60000; 22 | if (underMinutes < 30000) { 23 | time -= underMinutes; 24 | } else { 25 | time += 60000 - underMinutes; 26 | } 27 | 28 | return super.format(new Date(time), toAppendTo, pos); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClient.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.sqlserver.nativeclient; 2 | 3 | import jnr.ffi.Pointer; 4 | 5 | public interface NativeClient 6 | { 7 | static int SQL_NULL_DATA = -1; 8 | static int SQLCHARACTER = 0x2F; 9 | static int SQLINT1 = 0x30; 10 | static int SQLBIT = 0x32; 11 | static int SQLINT2 = 0x34; 12 | static int SQLINT4 = 0x38; 13 | static int SQLFLT8 = 0x3E; 14 | static int SQLFLT4 = 0x3B; 15 | static int SQLINT8 = 0x7F; 16 | 17 | static short FAIL = 0; 18 | static short SUCCEED = 1; 19 | static int DB_IN = 1; 20 | 21 | short bcp_initW(Pointer hdbc, Pointer szTable, Pointer szDataFile, Pointer szErrorFile, int eDirection); 22 | 23 | short bcp_bind(Pointer hdbc, 24 | Pointer pData, int cbIndicator, int cbData, 25 | Pointer pTerm, int cbTerm, 26 | int eDataType, int idxServerCol); 27 | 28 | short bcp_collen(Pointer hdbc, int cbData, int idxServerCol); 29 | 30 | short bcp_sendrow(Pointer hdbc); 31 | 32 | int bcp_batch(Pointer hdbc); 33 | 34 | int bcp_done(Pointer hdbc); 35 | } 36 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/main/java/org/embulk/output/sqlserver/nativeclient/ODBC.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.sqlserver.nativeclient; 2 | 3 | import jnr.ffi.Pointer; 4 | 5 | 6 | /** 7 | * sqlncli11.dll also has SQLXXX methods, but we should call those methods in odbc32.dll . 8 | */ 9 | public interface ODBC 10 | { 11 | static short SQL_SUCCESS = 0; 12 | static short SQL_SUCCESS_WITH_INFO = 1; 13 | 14 | static short SQL_HANDLE_ENV = 1; 15 | static short SQL_HANDLE_DBC = 2; 16 | 17 | static short SQL_ATTR_ODBC_VERSION = 200; 18 | 19 | static short SQL_COPT_SS_BASE = 1200; 20 | static short SQL_COPT_SS_BCP = SQL_COPT_SS_BASE + 19; 21 | 22 | static short SQL_DRIVER_NOPROMPT = 0; 23 | 24 | static long SQL_OV_ODBC3 = 3; 25 | static long SQL_BCP_ON = 1; 26 | 27 | static int SQL_IS_INTEGER = -6; 28 | static short SQL_NTS = -3; 29 | 30 | 31 | short SQLAllocHandle(short handleType, Pointer inputHandle, Pointer outputHandle); 32 | 33 | short SQLSetEnvAttr(Pointer environmentHandle, short attribute, Pointer value, int stringLength); 34 | 35 | short SQLSetConnectAttrW(Pointer hdbc, int fAttribute, Pointer rgbValue, int cbValue); 36 | 37 | short SQLDriverConnectW(Pointer hdbc, Pointer hwnd, 38 | Pointer szConnStrIn, short cchConnStrIn, 39 | Pointer szConnStrOut, short cchConnStrOutMax, Pointer pcchConnStrOut, 40 | short fDriverCompletion); 41 | 42 | short SQLFreeHandle(short handleType, Pointer handle); 43 | 44 | short SQLGetDiagRecW(short handleType, Pointer handle, short recNumber, 45 | Pointer sqlState, Pointer nativeErrorPtr, 46 | Pointer messageText, short bufferLength, Pointer textLengthPtr); 47 | } 48 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/main/java/org/embulk/output/sqlserver/setter/SQLServerByteColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.sqlserver.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | 6 | import org.embulk.output.jdbc.BatchInsert; 7 | import org.embulk.output.jdbc.JdbcColumn; 8 | import org.embulk.output.jdbc.setter.ByteColumnSetter; 9 | import org.embulk.output.jdbc.setter.DefaultValueSetter; 10 | 11 | public class SQLServerByteColumnSetter 12 | extends ByteColumnSetter 13 | { 14 | private static final short MIN_VALUE = 0; 15 | private static final short MAX_VALUE = 255; 16 | 17 | public SQLServerByteColumnSetter(BatchInsert batch, JdbcColumn column, 18 | DefaultValueSetter defaultValue) 19 | { 20 | super(batch, column, defaultValue); 21 | } 22 | 23 | @Override 24 | public void longValue(long v) throws IOException, SQLException 25 | { 26 | // SQLServer TINYINT value is from 0 to 255 27 | if (v > MAX_VALUE || v < MIN_VALUE) { 28 | defaultValue.setByte(); 29 | } else { 30 | batch.setShort((short) v); 31 | } 32 | } 33 | 34 | @Override 35 | public void stringValue(String v) throws IOException, SQLException 36 | { 37 | short sv; 38 | try { 39 | sv = Short.parseShort(v); 40 | } catch (NumberFormatException e) { 41 | defaultValue.setByte(); 42 | return; 43 | } 44 | longValue(sv); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/main/java/org/embulk/output/sqlserver/setter/SQLServerColumnSetterFactory.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.sqlserver.setter; 2 | 3 | import java.sql.Types; 4 | import java.time.ZoneId; 5 | import org.embulk.output.jdbc.BatchInsert; 6 | import org.embulk.output.jdbc.JdbcColumn; 7 | import org.embulk.output.jdbc.JdbcColumnOption; 8 | import org.embulk.output.jdbc.setter.ColumnSetter; 9 | import org.embulk.output.jdbc.setter.ColumnSetterFactory; 10 | import org.embulk.output.jdbc.setter.StringColumnSetter; 11 | 12 | public class SQLServerColumnSetterFactory 13 | extends ColumnSetterFactory 14 | { 15 | public SQLServerColumnSetterFactory(final BatchInsert batch, final ZoneId defaultTimeZone) 16 | { 17 | super(batch, defaultTimeZone); 18 | } 19 | 20 | @Override 21 | public ColumnSetter newCoalesceColumnSetter(JdbcColumn column, JdbcColumnOption option) 22 | { 23 | switch (column.getSqlType()) { 24 | case Types.TINYINT: 25 | return new SQLServerByteColumnSetter(batch, column, newDefaultValueSetter(column, option)); 26 | 27 | case Types.TIME: 28 | return new SQLServerSqlTimeColumnSetter(batch, column, newDefaultValueSetter(column, option), newCalendar(option)); 29 | 30 | default: 31 | return super.newCoalesceColumnSetter(column, option); 32 | } 33 | } 34 | 35 | @Override 36 | public ColumnSetter newColumnSetter(JdbcColumn column, JdbcColumnOption option) 37 | { 38 | switch (option.getValueType()) { 39 | case "byte": 40 | return new SQLServerByteColumnSetter(batch, column, newDefaultValueSetter(column, option)); 41 | 42 | case "time": 43 | return new SQLServerSqlTimeColumnSetter(batch, column, newDefaultValueSetter(column, option), newCalendar(option)); 44 | 45 | case "coerce": 46 | switch (column.getSimpleTypeName().toLowerCase()) { 47 | case "date": 48 | case "datetime2": 49 | case "time": 50 | case "sql_variant": 51 | case "datetimeoffset": 52 | // because jTDS driver, default JDBC driver for older embulk-output-sqlserver, returns Types.VARCHAR as JDBC type for these types. 53 | return new StringColumnSetter(batch, column, newDefaultValueSetter(column, option), newTimestampFormatter(option)); 54 | default: 55 | return super.newColumnSetter(column, option); 56 | } 57 | 58 | default: 59 | return super.newColumnSetter(column, option); 60 | } 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/main/java/org/embulk/output/sqlserver/setter/SQLServerSqlTimeColumnSetter.java: -------------------------------------------------------------------------------- 1 | package org.embulk.output.sqlserver.setter; 2 | 3 | import java.io.IOException; 4 | import java.sql.SQLException; 5 | import java.util.Calendar; 6 | import java.time.Instant; 7 | 8 | import org.embulk.output.jdbc.BatchInsert; 9 | import org.embulk.output.jdbc.JdbcColumn; 10 | import org.embulk.output.jdbc.setter.DefaultValueSetter; 11 | import org.embulk.output.jdbc.setter.SqlTimeColumnSetter; 12 | 13 | public class SQLServerSqlTimeColumnSetter 14 | extends SqlTimeColumnSetter 15 | { 16 | public SQLServerSqlTimeColumnSetter(BatchInsert batch, JdbcColumn column, 17 | DefaultValueSetter defaultValue, 18 | Calendar calendar) 19 | { 20 | super(batch, column, defaultValue, calendar); 21 | } 22 | 23 | @Override 24 | public void timestampValue(final Instant v) throws IOException, SQLException 25 | { 26 | // fractional precision of SQLServer TIME is 7, but that of java.sql.Time is only 3. 27 | batch.setSqlTimestamp(v, calendar); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/setup.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE TEST1 2 | CREATE TABLE TEST1 ( 3 | ID CHAR(4), 4 | TINYINT_ITEM TINYINT, 5 | SMALLINT_ITEM SMALLINT, 6 | INT_ITEM INT, 7 | BIGINT_ITEM BIGINT, 8 | BIT_ITEM BIT, 9 | DECIMAL_ITEM DECIMAL(12,2), 10 | NUMERIC_ITEM NUMERIC(5,3), 11 | SMALLMONEY_ITEM SMALLMONEY, 12 | MONEY_ITEM MONEY, 13 | REAL_ITEM REAL, 14 | FLOAT_ITEM FLOAT, 15 | CHAR_ITEM CHAR(4), 16 | VARCHAR_ITEM VARCHAR(8), 17 | TEXT_ITEM TEXT, 18 | NCHAR_ITEM NCHAR(4), 19 | NVARCHAR_ITEM NVARCHAR(8), 20 | NTEXT_ITEM NTEXT, 21 | DATE_ITEM DATE, 22 | DATETIME_ITEM DATETIME, 23 | DATETIME2_ITEM DATETIME2, 24 | DATETIME2_2_ITEM DATETIME2(2), 25 | SMALLDATETIME_ITEM SMALLDATETIME, 26 | TIME_ITEM TIME, 27 | TIME_2_ITEM TIME(2), 28 | PRIMARY KEY (ID) 29 | ); 30 | INSERT INTO TEST1(ID) VALUES('9999'); 31 | 32 | DROP TABLE TEST___Ａ１２３４５６７８９Ｂ１２３４５６７８９Ｃ１２３４５６７８９Ｄ１２３４５６７８９Ｅ１２３４５６７８９Ｆ１２３４５６７８９Ｇ１２３４５６７８９Ｈ１２３４５６７８９Ｉ１２３４５６７８９Ｊ１２３４５６７８９Ｋ１２３４５６７８９Ｌ１２３４５６７８９ 33 | CREATE TABLE TEST___Ａ１２３４５６７８９Ｂ１２３４５６７８９Ｃ１２３４５６７８９Ｄ１２３４５６７８９Ｅ１２３４５６７８９Ｆ１２３４５６７８９Ｇ１２３４５６７８９Ｈ１２３４５６７８９Ｉ１２３４５６７８９Ｊ１２３４５６７８９Ｋ１２３４５６７８９Ｌ１２３４５６７８９ ( 34 | ID CHAR(4), 35 | TINYINT_ITEM TINYINT, 36 | SMALLINT_ITEM SMALLINT, 37 | INT_ITEM INT, 38 | BIGINT_ITEM BIGINT, 39 | BIT_ITEM BIT, 40 | DECIMAL_ITEM DECIMAL(12,2), 41 | NUMERIC_ITEM NUMERIC(5,3), 42 | SMALLMONEY_ITEM SMALLMONEY, 43 | MONEY_ITEM MONEY, 44 | REAL_ITEM REAL, 45 | FLOAT_ITEM FLOAT, 46 | CHAR_ITEM CHAR(4), 47 | VARCHAR_ITEM VARCHAR(8), 48 | TEXT_ITEM TEXT, 49 | NCHAR_ITEM NCHAR(4), 50 | NVARCHAR_ITEM NVARCHAR(8), 51 | NTEXT_ITEM NTEXT, 52 | DATE_ITEM DATE, 53 | DATETIME_ITEM DATETIME, 54 | DATETIME2_ITEM DATETIME2, 55 | DATETIME2_2_ITEM DATETIME2(2), 56 | SMALLDATETIME_ITEM SMALLDATETIME, 57 | TIME_ITEM TIME, 58 | TIME_2_ITEM TIME(2), 59 | PRIMARY KEY (ID) 60 | ); 61 | INSERT INTO TEST___Ａ１２３４５６７８９Ｂ１２３４５６７８９Ｃ１２３４５６７８９Ｄ１２３４５６７８９Ｅ１２３４５６７８９Ｆ１２３４５６７８９Ｇ１２３４５６７８９Ｈ１２３４５６７８９Ｉ１２３４５６７８９Ｊ１２３４５６７８９Ｋ１２３４５６７８９Ｌ１２３４５６７８９(ID) VALUES('9999'); 62 | 63 | DROP TABLE TEST_MERGE1; 64 | CREATE TABLE TEST_MERGE1 ( 65 | ITEM1 INT, 66 | ITEM2 INT, 67 | ITEM3 VARCHAR(4), 68 | PRIMARY KEY(ITEM1, ITEM2) 69 | ); 70 | INSERT INTO TEST_MERGE1 VALUES(10, 20, 'A'); 71 | INSERT INTO TEST_MERGE1 VALUES(10, 21, 'B'); 72 | INSERT INTO TEST_MERGE1 VALUES(11, 20, 'C'); 73 | 74 | DROP TABLE TEST_MERGE2; 75 | CREATE TABLE TEST_MERGE2 ( 76 | ITEM1 INT, 77 | ITEM2 INT, 78 | ITEM3 VARCHAR(4) 79 | ); 80 | INSERT INTO TEST_MERGE2 VALUES(10, 20, 'A'); 81 | INSERT INTO TEST_MERGE2 VALUES(10, 21, 'B'); 82 | INSERT INTO TEST_MERGE2 VALUES(11, 20, 'C'); 83 | 84 | DROP TABLE TEST_MAX 85 | CREATE TABLE TEST_MAX ( 86 | ID INT, 87 | C1 VARCHAR(8000), 88 | C2 VARCHAR(MAX), 89 | C3 NVARCHAR(4000), 90 | C4 NVARCHAR(MAX), 91 | PRIMARY KEY (ID) 92 | ); -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test1.csv: -------------------------------------------------------------------------------- 1 | ID:string,TINYINT_ITEM:long,SMALLINT_ITEM:long,INT_ITEM:long,BIGINT_ITEM:long,BIT_ITEM:boolean,DECIMAL_ITEM:string,NUMERIC_ITEM:string,SMALLMONEY_ITEM:string,MONEY_ITEM:string,REAL_ITEM:double,FLOAT_ITEM:double,CHAR_ITEM:string,VARCHAR_ITEM:string,TEXT_ITEM:string,NCHAR_ITEM:string,NVARCHAR_ITEM:string,NTEXT_ITEM:string,DATE_ITEM:timestamp,DATETIME_ITEM:timestamp,DATETIME2_ITEM:timestamp,DATETIME2_2_ITEM:timestamp,SMALLDATETIME_ITEM:timestamp,TIME_ITEM:timestamp,TIME_2_ITEM:timestamp 2 | A001,0,1234,123456,12345678901,0,1.23,3.456,12.34,123.4567,0.1234567,0.12345678901234,a,b,c,A,B,C,2016-1-1 00:00:00.0 +0900,2017-1-1 1:2:3.123 +0900,2018-1-1 1:2:3.1234567 +0900,2019-1-1 1:2:3.12 +0900,2020-1-1 1:2:3.0 +0900,1970-01-01 3:4:5.1234567 +0900,1970-01-01 6:7:8.12 +0900 3 | A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9999000000,-999999999999000000,あい,あいうえ,あいうえお,かき,かきくけ,かきくけこ,2016-12-31 00:00:00.0 +0900,2017-12-31 23:59:59.997 +0900,2018-12-31 23:59:59.9999999 +0900,2019-12-31 23:59:59.99 +0900,2020-12-31 23:59:59.0 +0900,1970-01-01 23:59:59.9999999 +0900,1970-01-01 23:59:59.99 +0900 4 | A003,,,,,,,,,,,,,,,,,,,,,,,, 5 | A004,9,9,9,9,0,9,9,9,9,9,9,x,x,x,X,X,X,2016-1-1 00:00:00.0 +0900,2017-1-1 1:2:3.123 +0900,2018-1-1 1:2:3.1234567 +0900,2019-1-1 1:2:3.12 +0900,2020-1-1 1:2:3.0 +0900,1970-01-01 3:4:5.1234567 +0900,1970-01-01 6:7:8.12 +0900 6 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_expected.diff: -------------------------------------------------------------------------------- 1 | in: {} 2 | out: {} -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | mode: insert 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert_create_expected.csv: -------------------------------------------------------------------------------- 1 | A001,0,1234,123456,12345678901,0,1.23,3.456,12.34,123.4567,0.1234567,0.12345678901234,a,b,c,A,B,C,2015-12-31 15:00:00.0000000,2016-12-31 16:02:03.1230000,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.1200000,2019-12-31 16:02:03.0000000,1969-12-31 18:04:05.1234560,1969-12-31 21:07:08.1200000 2 | A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9999000000.0,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき,かきくけ,かきくけこ,2016-12-30 15:00:00.0000000,2017-12-31 14:59:59.9970000,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.9900000,2020-12-31 14:59:59.0000000,1970-01-01 14:59:59.9999990,1970-01-01 14:59:59.9900000 3 | A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL 4 | A004,9,9,9,9,0,9,9,9,9,9.0,9.0,x,x,x,X,X,X,2015-12-31 15:00:00.0000000,2016-12-31 16:02:03.1230000,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.1200000,2019-12-31 16:02:03.0000000,1969-12-31 18:04:05.1234560,1969-12-31 21:07:08.1200000 5 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert_direct.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | mode: insert_direct 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert_expected.csv: -------------------------------------------------------------------------------- 1 | 9999,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL 2 | A001,0,1234,123456,12345678901,0,1.23,3.456,12.3400,123.4567,0.1234567,0.12345678901234,a ,b,c,A ,B,C,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12 3 | A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9.9989996E+9,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき ,かきくけ,かきくけこ,2016-12-30,2017-12-31 14:59:59.997,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.99,2020-12-31 15:00:00,14:59:59.9999990,14:59:59.99 4 | A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL 5 | A004,9,9,9,9,0,9.00,9.000,9.0000,9.0000,9.0,9.0,x ,x,x,X ,X,X,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12 6 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_jtds.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | mode: insert 3 | driver_type: jtds 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_max.yml: -------------------------------------------------------------------------------- 1 | table: TEST_MAX 2 | mode: insert 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge.csv: -------------------------------------------------------------------------------- 1 | ITEM1:long,ITEM2:long,ITEM3:string 2 | 10,21,aa 3 | 11,10,bb 4 | 12,20,cc 5 | 10,22,dd 6 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge.yml: -------------------------------------------------------------------------------- 1 | table: TEST_MERGE1 2 | mode: merge 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge_expected.csv: -------------------------------------------------------------------------------- 1 | 10,20,A 2 | 10,21,aa 3 | 10,22,dd 4 | 11,10,bb 5 | 11,20,C 6 | 12,20,cc 7 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge_keys.yml: -------------------------------------------------------------------------------- 1 | table: TEST_MERGE2 2 | mode: merge 3 | merge_keys: [ITEM1,ITEM2] 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge_rule.yml: -------------------------------------------------------------------------------- 1 | table: TEST_MERGE1 2 | mode: merge 3 | merge_rule: ["ITEM3 = T.ITEM3 + S.ITEM3"] 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge_rule_expected.csv: -------------------------------------------------------------------------------- 1 | 10,20,A 2 | 10,21,Baa 3 | 10,22,dd 4 | 11,10,bb 5 | 11,20,C 6 | 12,20,cc 7 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_native_insert_direct.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_replace.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | mode: replace 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_replace_longname.yml: -------------------------------------------------------------------------------- 1 | table: TEST___Ａ１２３４５６７８９Ｂ１２３４５６７８９Ｃ１２３４５６７８９Ｄ１２３４５６７８９Ｅ１２３４５６７８９Ｆ１２３４５６７８９Ｇ１２３４５６７８９Ｈ１２３４５６７８９Ｉ１２３４５６７８９Ｊ１２３４５６７８９Ｋ１２３４５６７８９Ｌ１２３４５６７８９ 2 | mode: replace 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_string_timestamp.csv: -------------------------------------------------------------------------------- 1 | ID:string,TINYINT_ITEM:long,SMALLINT_ITEM:long,INT_ITEM:long,BIGINT_ITEM:long,BIT_ITEM:boolean,DECIMAL_ITEM:string,NUMERIC_ITEM:string,SMALLMONEY_ITEM:string,MONEY_ITEM:string,REAL_ITEM:double,FLOAT_ITEM:double,CHAR_ITEM:string,VARCHAR_ITEM:string,TEXT_ITEM:string,NCHAR_ITEM:string,NVARCHAR_ITEM:string,NTEXT_ITEM:string,DATE_ITEM:timestamp,DATETIME_ITEM:timestamp,DATETIME2_ITEM:string,DATETIME2_2_ITEM:timestamp,SMALLDATETIME_ITEM:timestamp,TIME_ITEM:string,TIME_2_ITEM:timestamp 2 | A001,0,1234,123456,12345678901,0,1.23,3.456,12.34,123.4567,0.1234567,0.12345678901234,a,b,c,A,B,C,2016-1-1 00:00:00.0 +0900,2017-1-1 1:2:3.123 +0900,2018/1/1 1:2:3.1234567,2019-1-1 1:2:3.12 +0900,2020-1-1 1:2:3.0 +0900,3:4:5.1234567,1970-01-01 6:7:8.12 +0900 3 | A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9999000000,-999999999999000000,あい,あいうえ,あいうえお,かき,かきくけ,かきくけこ,2016-12-31 00:00:00.0 +0900,2017-12-31 23:59:59.997 +0900,2018/12/31 23:59:59.9999999,2019-12-31 23:59:59.99 +0900,2020-12-31 23:59:59.0 +0900,23:59:59.9999999,1970-01-01 23:59:59.99 +0900 4 | A003,,,,,,,,,,,,,,,,,,,,,,,, 5 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_string_timestamp.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | mode: insert_direct 3 | column_options: 4 | DATETIME2_ITEM: {value_type: pass} 5 | TIME_ITEM: {value_type: pass} 6 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_string_timestamp_expected.csv: -------------------------------------------------------------------------------- 1 | 9999,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL 2 | A001,0,1234,123456,12345678901,0,1.23,3.456,12.3400,123.4567,0.1234567,0.12345678901234,a ,b,c,A ,B,C,2015-12-31,2016-12-31 16:02:03.123,2018-01-01 01:02:03.1234567,2018-12-31 16:02:03.12,2019-12-31 16:02:00,03:04:05.1234567,21:07:08.12 3 | A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9.9989996E+9,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき ,かきくけ,かきくけこ,2016-12-30,2017-12-31 14:59:59.997,2018-12-31 23:59:59.9999999,2019-12-31 14:59:59.99,2020-12-31 15:00:00,23:59:59.9999999,14:59:59.99 4 | A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL 5 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_truncate_insert.yml: -------------------------------------------------------------------------------- 1 | table: TEST1 2 | mode: truncate_insert 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_truncate_insert_expected.csv: -------------------------------------------------------------------------------- 1 | A001,0,1234,123456,12345678901,0,1.23,3.456,12.3400,123.4567,0.1234567,0.12345678901234,a ,b,c,A ,B,C,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12 2 | A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9.9989996E+9,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき ,かきくけ,かきくけこ,2016-12-30,2017-12-31 14:59:59.997,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.99,2020-12-31 15:00:00,14:59:59.9999990,14:59:59.99 3 | A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL 4 | A004,9,9,9,9,0,9.00,9.000,9.0000,9.0000,9.0,9.0,x ,x,x,X ,X,X,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12 5 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_bigint_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_BIGINT 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_bit_null.csv: -------------------------------------------------------------------------------- 1 | ID:string,value:long 2 | 1, 3 | 2,1 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_bit_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_BIT 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_bit_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1 ,NULL 2 | 2 ,1 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_char_null.csv: -------------------------------------------------------------------------------- 1 | ID:long,value:string 2 | 1, 3 | 2,xyz 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_char_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_CHAR 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_char_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1,NULL 2 | 2,xyz 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_date_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_DATE 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_date_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1,NULL 2 | 2,2017-12-23 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_datetime2_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_DATETIME2 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_datetime2_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1,NULL 2 | 2,2017-12-23 03:00:00.0000000 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_datetime_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_DATETIME 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_datetime_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1,NULL 2 | 2,2017-12-23 03:00:00.000 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_decimal_null.csv: -------------------------------------------------------------------------------- 1 | ID:string,value:long 2 | 1, 3 | 2,9999 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_decimal_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_DECIMAL 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_decimal_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1 ,NULL 2 | 2 ,9999.0000 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_float_null.csv: -------------------------------------------------------------------------------- 1 | ID:string,value:long 2 | 1, 3 | 2,99 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_float_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_FLOAT 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_float_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1 ,NULL 2 | 2 ,99.0 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_huge.csv: -------------------------------------------------------------------------------- 1 | dummy -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_huge.yml: -------------------------------------------------------------------------------- 1 | table: TEST_HUGE 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_int_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_INT 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_integer_null.csv: -------------------------------------------------------------------------------- 1 | ID:string,value:long 2 | 1, 3 | 2,99 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_integer_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1 ,NULL 2 | 2 ,99 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_money_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_MONEY 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_nchar_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_NCHAR 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_ntext_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_NTEXT 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_numeric_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_NUMERIC 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_nvarchar_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_NVARCHAR 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_real_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_REAL 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_smalldatetime_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_SMALLDATETIME 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_smalldatetime_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1,NULL 2 | 2,2017-12-23 03:00:00 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_smallint_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_SMALLINT 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_smallmoney_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_SMALLMONEY 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_text_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_TEXT 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_time_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_TIME 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_time_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1,NULL 2 | 2,03:00:00.0000000 3 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_timestamp_null.csv: -------------------------------------------------------------------------------- 1 | ID:long,value:timestamp 2 | 1, 3 | 2,2017-12-23 12:00:00.00 +0900 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_tinyint_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_TINYINT 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_varchar_null.csv: -------------------------------------------------------------------------------- 1 | ID:long,value:string 2 | 1, 3 | 2,xyz 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_varchar_null.yml: -------------------------------------------------------------------------------- 1 | table: TEST_VARCHAR 2 | mode: insert_direct 3 | insert_method: native 4 | -------------------------------------------------------------------------------- /embulk-output-sqlserver/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_varchar_null_expected.csv: -------------------------------------------------------------------------------- 1 | 1,NULL 2 | 2,xyz 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/embulk/embulk-output-jdbc/1a04c539bf879f8771245178f9bbc0d2cee71d30/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.4-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 48 | echo. 49 | echo Please set the JAVA_HOME variable in your environment to match the 50 | echo location of your Java installation. 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 62 | echo. 63 | echo Please set the JAVA_HOME variable in your environment to match the 64 | echo location of your Java installation. 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'embulk-output-jdbc-root' 2 | include 'embulk-output-jdbc' 3 | include 'embulk-output-mysql' 4 | include 'embulk-output-postgresql' 5 | include 'embulk-output-redshift' 6 | include 'embulk-output-sqlserver' 7 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/data/test.csv: -------------------------------------------------------------------------------- 1 | ID,NUM,STR,VARSTR,DT,DTTM0,DTTM3 2 | 1,123.40,test1,TEST1,2015-04-24,2015-04-24 01:02:03,2015-04-24 01:02:03.123 3 | 2,1234567890.12,test9999,TEST9999,2015-12-31,2015-12-31 23:59:59,2015-12-31 23:59:59.999 4 | 3,,,,,, 5 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/data/test_expected.txt: -------------------------------------------------------------------------------- 1 | ID NUM STR VARSTR DT DTTM0 DTTM3 2 | 1 123.40 test1 TEST1 2015-04-23 2015-04-24 01:02:03 2015-04-24 01:02:03.000 3 | 2 1234567890.12 test9999 TEST9999 2015-12-30 2015-12-31 23:59:59 2016-01-01 00:00:00.000 4 | 3 NULL NULL NULL NULL NULL NULL 5 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/data/test_replace_expected.txt: -------------------------------------------------------------------------------- 1 | ID NUM STR VARSTR DT DTTM0 DTTM3 2 | 1 123.40 test1 TEST1 2015-04-24 00:00:00 2015-04-24 01:02:03 2015-04-24 01:02:03 3 | 2 1234567890.12 test9999 TEST9999 2015-12-31 00:00:00 2015-12-31 23:59:59 2016-01-01 00:00:00 4 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/test.bat: -------------------------------------------------------------------------------- 1 | mysql -uTEST_USER -pXXXXXXXX -DTESTDB < test.sql 2 | 3 | CALL embulk run test.yml 4 | 5 | mysql -uTEST_USER -pXXXXXXXX -DTESTDB -e"SELECT * FROM EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test_expected.txt data/temp.txt" 8 | diff data/test_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-mysql FAILED!") 11 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/test.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS EMBULK_OUTPUT; 2 | 3 | CREATE TABLE EMBULK_OUTPUT ( 4 | ID INT, 5 | NUM DECIMAL(12,2), 6 | STR CHAR(8), 7 | VARSTR VARCHAR(8), 8 | DT DATE, 9 | DTTM0 DATETIME, 10 | DTTM3 DATETIME(3), 11 | PRIMARY KEY(ID) 12 | ); 13 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/test.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM0, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 17 | - {name: DTTM3, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | out: 19 | type: mysql 20 | host: localhost 21 | database: TESTDB 22 | user: TEST_USER 23 | password: XXXXXXXX 24 | table: EMBULK_OUTPUT 25 | mode: insert 26 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/test_temp_database.bat: -------------------------------------------------------------------------------- 1 | mysql -uTEST_USER -pXXXXXXXX -DTESTDB < test.sql 2 | 3 | CALL embulk run test_temp_database.yml 4 | 5 | mysql -uTEST_USER -pXXXXXXXX -DTESTDB -e"SELECT * FROM EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test_expected.txt data/temp.txt" 8 | diff data/test_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-mysql test_temp_database FAILED!") 11 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/test_temp_database.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM0, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 17 | - {name: DTTM3, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | out: 19 | type: mysql 20 | host: localhost 21 | database: TESTDB 22 | user: TEST_USER 23 | password: XXXXXXXX 24 | table: EMBULK_OUTPUT 25 | mode: insert 26 | temp_database: TESTDB2 27 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/test_temp_database_replace.bat: -------------------------------------------------------------------------------- 1 | mysql -uTEST_USER -pXXXXXXXX -DTESTDB < test.sql 2 | 3 | CALL embulk run test_temp_database_replace.yml 4 | 5 | mysql -uTEST_USER -pXXXXXXXX -DTESTDB -e"SELECT * FROM EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test_replace_expected.txt data/temp.txt" 8 | diff data/test_replace_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-mysql test_temp_database_replace FAILED!") 11 | -------------------------------------------------------------------------------- /test-scripts/mysql-output/test_temp_database_replace.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test_replace.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM0, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 17 | - {name: DTTM3, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | out: 19 | type: mysql 20 | host: localhost 21 | database: TESTDB 22 | user: TEST_USER 23 | password: XXXXXXXX 24 | table: EMBULK_OUTPUT 25 | mode: replace 26 | temp_database: TESTDB2 27 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/data/test-replace_expected.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/embulk/embulk-output-jdbc/1a04c539bf879f8771245178f9bbc0d2cee71d30/test-scripts/postgresql-output/data/test-replace_expected.txt -------------------------------------------------------------------------------- /test-scripts/postgresql-output/data/test.csv: -------------------------------------------------------------------------------- 1 | ID,NUM,STR,VARSTR,DT,DTTM0,DTTM3 2 | 1,123.40,test1,TEST1,2015-04-24,2015-04-24 01:02:03,2015-04-24 01:02:03.123 3 | 2,1234567890.12,test9999,TEST9999,2015-12-31,2015-12-31 23:59:59,2015-12-31 23:59:59.999 4 | 3,,,,,, 5 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/data/test2.csv: -------------------------------------------------------------------------------- 1 | ID,NUM,STR,VARSTR,DT,DTTM0,DTTM3 2 | 11,11,s11,v11,,, 3 | 12,12,s12,v12,,, 4 | 13,13,s13,v13,,, 5 | 14,14,s14,v14,,, 6 | 15,15,s15,v15,,, 7 | 16,16,s16,v16,,, 8 | 17,17,s17,v17,,, 9 | 18,18,s18,v18,,, 10 | 19,19,s19,v19,,, 11 | 20,20,s20,v20,,, 12 | 21,21,s21,v21,,, 13 | 22,22,s22,v22,,, 14 | 23,23,s23,v23,,, 15 | 24,24,s24,v24,,, 16 | 25,25,s25,v25,,, 17 | 26,26,s26,v26,,, 18 | 27,27,s27,v27,,, 19 | 28,28,s28,v28,,, 20 | 29,29,s29,v29,,, 21 | 30,30,s30,v30,,, 22 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/data/test_expected.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/embulk/embulk-output-jdbc/1a04c539bf879f8771245178f9bbc0d2cee71d30/test-scripts/postgresql-output/data/test_expected.txt -------------------------------------------------------------------------------- /test-scripts/postgresql-output/test-temp-schema-replace.bat: -------------------------------------------------------------------------------- 1 | SETLOCAL 2 | SET PGPASSWORD=XXXXXXXX 3 | "C:\Program Files\PostgreSQL\9.4\bin\psql.exe" -d testdb -U test_user -w -f test.sql 4 | ENDLOCAL 5 | 6 | CALL embulk run test-temp-schema-replace.yml 7 | 8 | SETLOCAL 9 | SET PGPASSWORD=XXXXXXXX 10 | "C:\Program Files\PostgreSQL\9.4\bin\psql.exe" -d testdb -U test_user -w -c "SELECT * FROM embulk_output" > data/temp.txt 11 | ENDLOCAL 12 | 13 | echo "diff data/test-replace_expected.txt data/temp.txt" 14 | diff data/test-replace_expected.txt data/temp.txt 15 | 16 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-postgresql test-temp-schema-replace FAILED!") 17 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/test-temp-schema-replace.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: id, type: long} 12 | - {name: num, type: string} 13 | - {name: str, type: string} 14 | - {name: varstr, type: string} 15 | - {name: dt, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: dttm0, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 17 | - {name: dttm3, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | out: 19 | type: postgresql 20 | host: localhost 21 | database: testdb 22 | user: test_user 23 | password: XXXXXXXX 24 | table: embulk_output 25 | mode: replace 26 | temp_schema: test 27 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/test-temp-schema.bat: -------------------------------------------------------------------------------- 1 | SETLOCAL 2 | SET PGPASSWORD=XXXXXXXX 3 | "C:\Program Files\PostgreSQL\9.4\bin\psql.exe" -d testdb -U test_user -w -f test.sql 4 | ENDLOCAL 5 | 6 | CALL embulk run test-temp-schema.yml 7 | 8 | SETLOCAL 9 | SET PGPASSWORD=XXXXXXXX 10 | "C:\Program Files\PostgreSQL\9.4\bin\psql.exe" -d testdb -U test_user -w -c "SELECT * FROM embulk_output" > data/temp.txt 11 | ENDLOCAL 12 | 13 | echo "diff data/test_expected.txt data/temp.txt" 14 | diff data/test_expected.txt data/temp.txt 15 | 16 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-postgresql test-temp-schema FAILED!") 17 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/test-temp-schema.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: id, type: long} 12 | - {name: num, type: string} 13 | - {name: str, type: string} 14 | - {name: varstr, type: string} 15 | - {name: dt, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: dttm0, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 17 | - {name: dttm3, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | out: 19 | type: postgresql 20 | host: localhost 21 | database: testdb 22 | user: test_user 23 | password: XXXXXXXX 24 | table: embulk_output 25 | mode: insert 26 | temp_schema: test 27 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/test.bat: -------------------------------------------------------------------------------- 1 | SETLOCAL 2 | SET PGPASSWORD=XXXXXXXX 3 | "C:\Program Files\PostgreSQL\9.4\bin\psql.exe" -d testdb -U test_user -w -f test.sql 4 | ENDLOCAL 5 | 6 | CALL embulk run test.yml 7 | 8 | SETLOCAL 9 | SET PGPASSWORD=XXXXXXXX 10 | "C:\Program Files\PostgreSQL\9.4\bin\psql.exe" -d testdb -U test_user -w -c "SELECT * FROM embulk_output" > data/temp.txt 11 | ENDLOCAL 12 | 13 | echo "diff data/test_expected.txt data/temp.txt" 14 | diff data/test_expected.txt data/temp.txt 15 | 16 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-postgresql test FAILED!") 17 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/test.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS embulk_output; 2 | 3 | CREATE TABLE embulk_output ( 4 | id int, 5 | num decimal(12,2), 6 | str char(8), 7 | varstr varchar(8), 8 | dt date, 9 | dttm0 timestamp, 10 | dttm3 timestamp(3), 11 | primary key(ID) 12 | ); 13 | -------------------------------------------------------------------------------- /test-scripts/postgresql-output/test.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: id, type: long} 12 | - {name: num, type: string} 13 | - {name: str, type: string} 14 | - {name: varstr, type: string} 15 | - {name: dt, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: dttm0, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 17 | - {name: dttm3, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | out: 19 | type: postgresql 20 | host: localhost 21 | database: testdb 22 | user: test_user 23 | password: XXXXXXXX 24 | table: embulk_output 25 | mode: insert 26 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/data/test-replace_expected.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/embulk/embulk-output-jdbc/1a04c539bf879f8771245178f9bbc0d2cee71d30/test-scripts/sqlserver-output/data/test-replace_expected.txt -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/data/test.csv: -------------------------------------------------------------------------------- 1 | ID,NUM,STR,VARSTR,DT,DTTM,DTTM2,SDTTM 2 | 1,123.40,test1,TEST1,2015-04-24,2015-04-24 01:02:03.123,2015-04-24 01:02:03.1234567,2015-04-24 01:02:03 3 | 2,1234567890.12,test9999,TEST9999,2015-12-31,2015-12-31 23:59:59.999,2015-12-31 23:59:59.9999999,2015-12-31 23:59:59 4 | 3,,,,,,, 5 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/data/test_expected.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/embulk/embulk-output-jdbc/1a04c539bf879f8771245178f9bbc0d2cee71d30/test-scripts/sqlserver-output/data/test_expected.txt -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-jtds.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test.sql 2 | 3 | CALL embulk run test-jtds.yml 4 | 5 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -Q "SELECT * FROM EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test_expected.txt data/temp.txt" 8 | diff data/test_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-sqlserver FAILED!") -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-jtds.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 17 | - {name: DTTM2, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | - {name: SDTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 19 | out: 20 | type: sqlserver 21 | host: localhost 22 | host: localhost 23 | port: 1433 24 | database: TESTDB 25 | user: TEST_USER 26 | password: XXXXXXXX 27 | table: EMBULK_OUTPUT 28 | mode: insert 29 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-native.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test.sql 2 | 3 | CALL embulk run test-native.yml 4 | 5 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -Q "SELECT * FROM EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test_expected.txt data/temp.txt" 8 | diff data/test_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-sqlserver (native) FAILED!") -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-native.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 17 | - {name: DTTM2, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | - {name: SDTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 19 | out: 20 | type: sqlserver 21 | host: localhost 22 | driver_path: driver/sqljdbc41.jar 23 | host: localhost 24 | port: 1433 25 | database: TESTDB 26 | user: TEST_USER 27 | password: XXXXXXXX 28 | table: EMBULK_OUTPUT 29 | mode: insert 30 | insert_method: native 31 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-replace-schema.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test.sql 2 | 3 | CALL embulk run test-replace-schema.yml 4 | 5 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -Q "SELECT * FROM test.EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test-replace_expected.txt data/temp.txt" 8 | diff data/test-replace_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-sqlserver FAILED!") -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-replace-schema.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 17 | - {name: DTTM2, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | - {name: SDTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 19 | out: 20 | type: sqlserver 21 | host: localhost 22 | driver_path: driver/sqljdbc41.jar 23 | host: localhost 24 | port: 1433 25 | database: TESTDB 26 | user: TEST_USER 27 | password: XXXXXXXX 28 | table: EMBULK_OUTPUT 29 | schema: test 30 | mode: replace 31 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-replace-temp-schema.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test.sql 2 | 3 | CALL embulk run test-replace-temp-schema.yml 4 | 5 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -Q "SELECT * FROM EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test-replace_expected.txt data/temp.txt" 8 | diff data/test-replace_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-sqlserver FAILED!") -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-replace-temp-schema.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 17 | - {name: DTTM2, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | - {name: SDTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 19 | out: 20 | type: sqlserver 21 | host: localhost 22 | driver_path: driver/sqljdbc41.jar 23 | host: localhost 24 | port: 1433 25 | database: TESTDB 26 | user: TEST_USER 27 | password: XXXXXXXX 28 | table: EMBULK_OUTPUT 29 | mode: replace 30 | temp_schema: test 31 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-schema.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test-schema.sql 2 | 3 | CALL embulk run test-schema.yml 4 | 5 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -Q "SELECT * FROM test.EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test_expected.txt data/temp.txt" 8 | diff data/test_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-sqlserver FAILED!") -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE test.EMBULK_OUTPUT; 2 | 3 | CREATE TABLE test.EMBULK_OUTPUT ( 4 | ID INT, 5 | NUM DECIMAL(12,2), 6 | STR CHAR(8), 7 | VARSTR VARCHAR(8), 8 | DT DATE, 9 | DTTM DATETIME, 10 | DTTM2 DATETIME2, 11 | SDTTM SMALLDATETIME, 12 | PRIMARY KEY(ID) 13 | ); 14 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-schema.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 17 | - {name: DTTM2, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | - {name: SDTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 19 | out: 20 | type: sqlserver 21 | host: localhost 22 | driver_path: driver/sqljdbc41.jar 23 | host: localhost 24 | port: 1433 25 | database: TESTDB 26 | schema: test 27 | user: TEST_USER 28 | password: XXXXXXXX 29 | table: EMBULK_OUTPUT 30 | mode: insert 31 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-temp-schema.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test.sql 2 | 3 | CALL embulk run test-temp-schema.yml 4 | 5 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -Q "SELECT * FROM EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test_expected.txt data/temp.txt" 8 | diff data/test_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-sqlserver FAILED!") -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test-temp-schema.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 17 | - {name: DTTM2, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | - {name: SDTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 19 | out: 20 | type: sqlserver 21 | host: localhost 22 | driver_path: driver/sqljdbc41.jar 23 | host: localhost 24 | port: 1433 25 | database: TESTDB 26 | user: TEST_USER 27 | password: XXXXXXXX 28 | table: EMBULK_OUTPUT 29 | mode: insert 30 | temp_schema: test 31 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test.bat: -------------------------------------------------------------------------------- 1 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -i test.sql 2 | 3 | CALL embulk run test.yml 4 | 5 | osql -d TESTDB -U TEST_USER -P XXXXXXXX -Q "SELECT * FROM EMBULK_OUTPUT" > data/temp.txt 6 | 7 | echo "diff data/test_expected.txt data/temp.txt" 8 | diff data/test_expected.txt data/temp.txt 9 | 10 | IF "%ERRORLEVEL%" == "0" (ECHO "OK!") ELSE (ECHO "embulk-output-sqlserver FAILED!") -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE EMBULK_OUTPUT; 2 | 3 | CREATE TABLE EMBULK_OUTPUT ( 4 | ID INT, 5 | NUM DECIMAL(12,2), 6 | STR CHAR(8), 7 | VARSTR VARCHAR(8), 8 | DT DATE, 9 | DTTM DATETIME, 10 | DTTM2 DATETIME2, 11 | SDTTM SMALLDATETIME, 12 | PRIMARY KEY(ID) 13 | ); 14 | -------------------------------------------------------------------------------- /test-scripts/sqlserver-output/test.yml: -------------------------------------------------------------------------------- 1 | in: 2 | type: file 3 | path_prefix: 'data/test.csv' 4 | parser: 5 | charset: UTF-8 6 | newline: CRLF 7 | type: csv 8 | delimiter: ',' 9 | header_line: true 10 | columns: 11 | - {name: ID, type: long} 12 | - {name: NUM, type: string} 13 | - {name: STR, type: string} 14 | - {name: VARSTR, type: string} 15 | - {name: DT, type: timestamp, format: '%Y-%m-%d', timezone: "+0900"} 16 | - {name: DTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 17 | - {name: DTTM2, type: timestamp, format: '%Y-%m-%d %H:%M:%S.%N', timezone: "+0900"} 18 | - {name: SDTTM, type: timestamp, format: '%Y-%m-%d %H:%M:%S', timezone: "+0900"} 19 | out: 20 | type: sqlserver 21 | host: localhost 22 | driver_path: driver/sqljdbc41.jar 23 | host: localhost 24 | port: 1433 25 | database: TESTDB 26 | user: TEST_USER 27 | password: XXXXXXXX 28 | table: EMBULK_OUTPUT 29 | mode: insert 30 | -------------------------------------------------------------------------------- /test-scripts/test-output.bat: -------------------------------------------------------------------------------- 1 | set LOG=%~dp0test-output.log 2 | echo log file = %LOG% 3 | del %LOG% 4 | 5 | cd mysql-output 6 | echo "mysql-output/test.bat" 7 | call test.bat >> %LOG% 8 | echo "mysql-output/test_temp_database.bat" 9 | call test_temp_database.bat >> %LOG% 10 | echo "mysql-output/test_temp_database-replace.bat" 11 | call test_temp_database_replace.bat >> %LOG% 12 | cd .. 13 | 14 | cd postgresql-output 15 | echo "postgresql-output/test.bat" 16 | call test.bat >> %LOG% 17 | echo "postgresql-output/test-temp-schema.bat" 18 | call test-temp-schema.bat >> %LOG% 19 | echo "postgresql-output/test-temp-schema-replace.bat" 20 | call test-temp-schema-replace.bat >> %LOG% 21 | cd .. 22 | 23 | cd sqlserver-output 24 | echo "sqlserver-output/test.bat" 25 | call test.bat >> %LOG% 26 | echo "sqlserver-output/test-jtds.bat" 27 | call test-jtds.bat >> %LOG% 28 | echo "sqlserver-output/test-native.bat" 29 | call test-native.bat >> %LOG% 30 | echo "sqlserver-output/test-temp-schema.bat" 31 | call test-temp-schema.bat >> %LOG% 32 | echo "sqlserver-output/test-replace-temp-schema.bat" 33 | call test-replace-temp-schema.bat >> %LOG% 34 | cd .. 35 | 36 | grep "FAILED" %LOG% 37 | 38 | IF "%ERRORLEVEL%" == "0" (ECHO "FAILED!") ELSE (ECHO "OK!") 39 | --------------------------------------------------------------------------------