├── .idea
├── .name
├── copyright
│ └── profiles_settings.xml
├── scopes
│ └── scope_settings.xml
├── ant.xml
├── vcs.xml
├── encodings.xml
├── modules.xml
├── libraries
│ ├── Maven__junit_junit_4_10.xml
│ ├── Maven__stax_stax_api_1_0_1.xml
│ ├── Maven__io_netty_netty_3_5_5_Final.xml
│ ├── Maven__com_google_code_gson_gson_2_2_2.xml
│ ├── Maven__net_spy_spymemcached_2_9_0.xml
│ ├── Maven__org_hamcrest_hamcrest_core_1_1.xml
│ ├── Maven__commons_codec_commons_codec_1_5.xml
│ ├── Maven__org_codehaus_jettison_jettison_1_1.xml
│ ├── Maven__couchbase_couchbase_client_1_1_7.xml
│ ├── Maven__org_apache_httpcomponents_httpcore_4_1_1.xml
│ └── Maven__org_apache_httpcomponents_httpcore_nio_4_1_1.xml
├── compiler.xml
├── misc.xml
├── uiDesigner.xml
└── workspace.xml
├── .gitignore
├── sql-importer-lib
├── src
│ ├── main
│ │ ├── resources
│ │ │ └── sample-import.properties
│ │ ├── assembly
│ │ │ └── assembly.xml
│ │ └── java
│ │ │ └── com
│ │ │ └── couchbase
│ │ │ └── util
│ │ │ └── SqlImporter.java
│ └── test
│ │ └── java
│ │ └── com
│ │ └── couchbase
│ │ └── util
│ │ └── importer
│ │ └── SqlImporterTest.java
├── couchbase-sql-importer (2).iml
├── README.MD
└── pom.xml
├── pom.xml
└── README.md
/.idea/.name:
--------------------------------------------------------------------------------
1 | couchbase-sql-importer.parent
--------------------------------------------------------------------------------
/.idea/copyright/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/.idea/scopes/scope_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
--------------------------------------------------------------------------------
/.idea/ant.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/encodings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # General Ignores
2 | *~
3 | .classpath
4 | .DS_Store
5 | .project
6 | build/
7 | bin/
8 | target/
9 | log/
10 |
11 | # IDE Support
12 | # IntelliJ specific files/directories
13 | .idea/
14 | *.ipr
15 | *.iws
16 | *.iml
17 | atlassian-ide-plugin.xml
18 |
19 | # Eclipse
20 | .factorypath
21 | .settings/
22 | .externalToolBuilders
23 |
24 | # NetBeans
25 | .nbattrs
26 |
--------------------------------------------------------------------------------
/sql-importer-lib/src/main/resources/sample-import.properties:
--------------------------------------------------------------------------------
1 | ## SQL Information ##
2 | sql.connection=jdbc:mysql://192.168.99.19:3306/world
3 | sql.username=root
4 | sql.password=password
5 |
6 | ## Couchbase Information ##
7 | cb.uris=http://localhost:8091/pools
8 | cb.bucket=default
9 | cb.password=
10 |
11 | ## Import information
12 | import.tables=ALL
13 | import.createViews=true
14 | import.typefield=type
15 | import.fieldcase=lower
16 |
17 |
18 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__junit_junit_4_10.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__stax_stax_api_1_0_1.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__io_netty_netty_3_5_5_Final.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__com_google_code_gson_gson_2_2_2.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__net_spy_spymemcached_2_9_0.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__org_hamcrest_hamcrest_core_1_1.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__commons_codec_commons_codec_1_5.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__org_codehaus_jettison_jettison_1_1.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__couchbase_couchbase_client_1_1_7.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_4_1_1.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/sql-importer-lib/src/main/assembly/assembly.xml:
--------------------------------------------------------------------------------
1 |
2 | all
3 |
4 | jar
5 |
6 | false
7 |
8 |
9 | true
10 | runtime
11 | false
12 |
13 |
14 |
15 |
16 | ${project.build.outputDirectory}
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/.idea/libraries/Maven__org_apache_httpcomponents_httpcore_nio_4_1_1.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/sql-importer-lib/src/test/java/com/couchbase/util/importer/SqlImporterTest.java:
--------------------------------------------------------------------------------
1 | package com.couchbase.util.importer;
2 |
3 | import org.junit.*;
4 |
5 | import java.net.URI;
6 | import java.util.LinkedList;
7 | import java.util.List;
8 | import java.util.concurrent.TimeUnit;
9 | import java.util.logging.Logger;
10 |
11 | /**
12 | * Created with IntelliJ IDEA.
13 | * User: tgrall
14 | * Date: 6/29/13
15 | * Time: 6:41 AM
16 | * To change this template use File | Settings | File Templates.
17 | */
18 | public class SqlImporterTest {
19 |
20 |
21 |
22 | @BeforeClass
23 | public static void setUpClass() {
24 | }
25 |
26 | @AfterClass
27 | public static void tearDownClass() {
28 | }
29 |
30 | @Before
31 | public void setUp() {
32 | }
33 |
34 | @After
35 | public void tearDown() {
36 | }
37 |
38 | @Test
39 | public void testOfSetOperationWithVersion() {
40 |
41 |
42 |
43 | }
44 |
45 |
46 | }
47 |
--------------------------------------------------------------------------------
/.idea/compiler.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
1 |
3 | 4.0.0
4 |
5 | com.couchbase.util
6 | couchbase-sql-importer.parent
7 | 1.0-SNAPSHOT
8 | pom
9 | Couchbase SQL Importer
10 |
11 |
12 | ${project.build.directory}/endorsed
13 | UTF-8
14 |
15 |
16 |
17 |
18 | couchbase
19 | Couchbase Maven Repository
20 | default
21 | http://files.couchbase.com/maven2/
22 |
23 | false
24 |
25 |
26 |
27 |
28 |
29 |
30 | couchbase
31 | couchbase-client
32 | 1.1.7
33 |
34 |
35 | com.google.code.gson
36 | gson
37 | 2.2.2
38 |
39 |
40 | junit
41 | junit
42 | 4.10
43 | test
44 |
45 |
46 |
47 |
48 |
49 | sql-importer-lib
50 |
51 |
52 |
53 |
54 |
55 |
--------------------------------------------------------------------------------
/sql-importer-lib/couchbase-sql-importer (2).iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/sql-importer-lib/README.MD:
--------------------------------------------------------------------------------
1 | Couchbase SQL Importer
2 | ===
3 |
4 | This tool allows you to copy the content of a your tables into Couchbase.
5 |
6 | The import
7 |
8 | * Import all rows as JSON document. All table columns are JSON attributes.
9 | * Optionnaly it is possible to create views, the import tool create 2 types of views:
10 | * A view all/by_type that allows to query document by type (table)
11 | * Views for each type (table) to query on Primary Key (type/by_pk)
12 | * All the views are using the _count reduce function.
13 |
14 | If you do not want to build it from source, you can download the JAR [here](http://goo.gl/IF89e)
15 |
16 |
17 | Usage
18 | -----
19 |
20 | 1. Configure an import.properties file with all the parameters
21 |
22 | ## SQL Information ##
23 | sql.connection=jdbc:mysql://192.168.99.19:3306/world
24 | sql.username=root
25 | sql.password=password
26 |
27 | ## Couchbase Information ##
28 | cb.uris=http://localhost:8091/pools
29 | cb.bucket=default
30 | cb.password=
31 |
32 | ## Import information
33 | import.tables=ALL
34 | import.createViews=true
35 | import.typefield=type
36 | import.fieldcase=lower
37 |
38 |
39 | 2. Download the JDBC driver for your database
40 |
41 | 3. Run the following command
42 |
43 | java -cp "./CouchbaseSqlImporter.jar:./mysql-connector-java-5.1.25-bin.jar" com.couchbase.util.SqlImporter import.properties
44 |
45 |
46 |
47 | Notes
48 | -----
49 | This tool is a small utility that I have developed in few hours as a demonstrator.
50 |
51 | I have tested the tool mainly on MySQL with some sample schemas: World (few hundreds of records), Employees (4 millions records);
52 | but I have not tried to optimize the import "speed" (for example multithreading, divide select is chunks, ...).
53 |
54 | I addition to the tests that I need to do with multiple databases types, I also need to create a test suite.
55 |
56 | **To do**
57 |
58 |
59 | * Implement a test suite
60 | * Provide "Report/Logging"
61 | * Create Document keys using PK columns
62 | * Error/Log for long keys (>250char)
63 | * Test multiple databases, at least Oracle
64 |
65 |
66 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | Couchbase SQL Importer
2 | ===
3 |
4 | This tool allows you to copy the content of a your tables into Couchbase.
5 |
6 | The import
7 |
8 | * Import all rows as JSON document. All table columns are JSON attributes.
9 | * Optionnaly it is possible to create views, the import tool create 2 types of views:
10 | * A view all/by_type that allows to query document by type (table)
11 | * Views for each type (table) to query on Primary Key (type/by_pk)
12 | * All the views are using the _count reduce function.
13 |
14 | If you do not want to build it from source, you can download the JAR [here](http://goo.gl/IF89e)
15 |
16 |
17 | Usage
18 | -----
19 |
20 | 1. Configure an import.properties file with all the parameters
21 |
22 | ## SQL Information ##
23 | sql.connection=jdbc:mysql://192.168.99.19:3306/world?zeroDateTimeBehavior=convertToNull
24 | sql.username=root
25 | sql.password=password
26 |
27 | ## Couchbase Information ##
28 | cb.uris=http://localhost:8091/pools
29 | cb.bucket=default
30 | cb.password=
31 |
32 | ## Import information
33 | import.tables=ALL
34 | import.createViews=true
35 | import.typefield=type
36 | import.fieldcase=lower
37 |
38 |
39 | 2. Download the JDBC driver for your database
40 |
41 | 3. Run the following command
42 |
43 | java -cp "./CouchbaseSqlImporter.jar:./mysql-connector-java-5.1.25-bin.jar" com.couchbase.util.SqlImporter import.properties
44 |
45 |
46 |
47 | Notes
48 | -----
49 | This tool is a small utility that I have developed in few hours as a demonstrator.
50 |
51 | I have tested the tool mainly on MySQL with some sample schemas: World (few hundreds of records), Employees (4 millions records);
52 | but I have not tried to optimize the import "speed" (for example multithreading, divide select is chunks, ...).
53 |
54 | I addition to the tests that I need to do with multiple databases types, I also need to create a test suite.
55 |
56 | **To do**
57 |
58 |
59 | * Implement a test suite
60 | * Provide "Report/Logging"
61 | * Create Document keys using PK columns
62 | * Error/Log for long keys (>250char)
63 | * Test multiple databases, at least Oracle
64 |
65 |
66 |
--------------------------------------------------------------------------------
/sql-importer-lib/pom.xml:
--------------------------------------------------------------------------------
1 |
3 | 4.0.0
4 |
5 | com.couchbase.util
6 | couchbase-sql-importer.parent
7 | 1.0-SNAPSHOT
8 |
9 | com.couchbase.util
10 | couchbase-sql-importer
11 | 1.0-SNAPSHOT
12 | jar
13 |
14 | Couchbase SQL Importer
15 |
16 |
17 |
18 |
19 |
20 | couchbase
21 | couchbase-client
22 | 1.1.7
23 |
24 |
25 | com.google.code.gson
26 | gson
27 | 2.2.2
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 | maven-assembly-plugin
41 | 2.2-beta-2
42 |
43 |
44 | create-executable-jar
45 | package
46 |
47 | single
48 |
49 |
50 | CouchbaseSqlImporter
51 | false
52 |
53 | src/main/assembly/assembly.xml
54 |
55 |
56 |
57 | com.couchbase.util.SqlImporter
58 |
59 |
60 | .
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
52 |
55 |
56 |
57 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
164 |
165 |
166 |
167 |
168 |
169 |
170 | localhost
171 | 5050
172 |
173 |
174 |
175 |
176 |
177 |
178 | 1.6
179 |
180 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
--------------------------------------------------------------------------------
/.idea/uiDesigner.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | -
6 |
7 |
8 | -
9 |
10 |
11 | -
12 |
13 |
14 | -
15 |
16 |
17 | -
18 |
19 |
20 |
21 |
22 |
23 | -
24 |
25 |
26 |
27 |
28 |
29 | -
30 |
31 |
32 |
33 |
34 |
35 | -
36 |
37 |
38 |
39 |
40 |
41 | -
42 |
43 |
44 |
45 |
46 | -
47 |
48 |
49 |
50 |
51 | -
52 |
53 |
54 |
55 |
56 | -
57 |
58 |
59 |
60 |
61 | -
62 |
63 |
64 |
65 |
66 | -
67 |
68 |
69 |
70 |
71 | -
72 |
73 |
74 | -
75 |
76 |
77 |
78 |
79 | -
80 |
81 |
82 |
83 |
84 | -
85 |
86 |
87 |
88 |
89 | -
90 |
91 |
92 |
93 |
94 | -
95 |
96 |
97 |
98 |
99 | -
100 |
101 |
102 | -
103 |
104 |
105 | -
106 |
107 |
108 | -
109 |
110 |
111 | -
112 |
113 |
114 |
115 |
116 | -
117 |
118 |
119 | -
120 |
121 |
122 |
123 |
124 |
125 |
126 |
--------------------------------------------------------------------------------
/sql-importer-lib/src/main/java/com/couchbase/util/SqlImporter.java:
--------------------------------------------------------------------------------
1 | package com.couchbase.util;
2 |
3 | import com.couchbase.client.CouchbaseClient;
4 | import com.couchbase.client.protocol.views.DesignDocument;
5 | import com.couchbase.client.protocol.views.ViewDesign;
6 | import com.google.gson.Gson;
7 |
8 | import java.io.FileInputStream;
9 | import java.net.URI;
10 | import java.sql.*;
11 | import java.util.*;
12 | import java.util.concurrent.TimeUnit;
13 | import java.util.logging.ConsoleHandler;
14 | import java.util.logging.Handler;
15 | import java.util.logging.Level;
16 | import java.util.logging.Logger;
17 |
18 | public class SqlImporter {
19 |
20 |
21 | private static final String COUCHBASE_URIS = "cb.uris";
22 | private static final String COUCHBASE_BUCKET = "cb.bucket";
23 | private static final String COUCHBASE_PASSWORD = "cb.password";
24 |
25 | private static final String SQL_CONN_STRING = "sql.connection";
26 | private static final String SQL_USER = "sql.username";
27 | private static final String SQL_PASSWORD = "sql.password";
28 |
29 | private static final String TABLES_LIST = "import.tables";
30 | private static final String CREATE_VIEWS = "import.createViews";
31 | private static final String TYPE_FIELD = "import.typefield";
32 | private static final String TYPE_CASE = "import.fieldcase";
33 |
34 | private CouchbaseClient couchbaseClient = null;
35 | private Connection connection = null;
36 |
37 | // JDBC connection
38 | private String sqlConnString = null;
39 | private String sqlUser = null;
40 | private String sqlPassword = null;
41 |
42 | // Couchbase information
43 | private List uris = new ArrayList();
44 | private String bucket = "default";
45 | private String defaultUri = "http://127.0.0.1:8091/pools";
46 | private String password = "";
47 |
48 | // import options
49 | private String typeField = null;
50 | private String typeFieldCase = null;
51 | private boolean createTableViewEnable = true;
52 | private String[] tableList = null;
53 |
54 |
55 | public static void main(String[] args) {
56 |
57 |
58 | System.out.println("\n\n");
59 | System.out.println("############################################");
60 | System.out.println("# COUCHBASE SQL IMPORTER #");
61 | System.out.println("############################################\n\n");
62 |
63 |
64 | if (args == null || args.length == 0) {
65 | System.out.println("ERROR : You must specify a propeprty file when you run this command\n\n");
66 | System.exit(0);
67 | }
68 |
69 | SqlImporter importer = new SqlImporter();
70 | try {
71 |
72 |
73 | // remove log info from Couchbase
74 | Properties systemProperties = System.getProperties();
75 | systemProperties.put("net.spy.log.LoggerImpl", "net.spy.memcached.compat.log.SunLogger");
76 | System.setProperties(systemProperties);
77 | Logger logger = Logger.getLogger("com.couchbase.client");
78 | logger.setLevel(Level.WARNING);
79 | for (Handler h : logger.getParent().getHandlers()) {
80 | if (h instanceof ConsoleHandler) {
81 | h.setLevel(Level.WARNING);
82 | }
83 | }
84 |
85 |
86 | importer.setup(args[0]);
87 | importer.importData();
88 | importer.shutdown();
89 |
90 |
91 | } catch (Exception e) {
92 | e.printStackTrace();
93 | }
94 |
95 | System.out.println("\n\n FINISHED");
96 | System.out.println("############################################");
97 | System.out.println("\n\n");
98 |
99 |
100 | }
101 |
102 | private void setup(String fileName) {
103 | try {
104 | Properties prop = new Properties();
105 | prop.load(new FileInputStream(fileName));
106 |
107 | if (prop.containsKey(COUCHBASE_URIS)) {
108 | String[] uriStrings = prop.getProperty(COUCHBASE_URIS).split(",");
109 | for (int i = 0; i < uriStrings.length; i++) {
110 | uris.add(new URI(uriStrings[i]));
111 | }
112 |
113 | } else {
114 | uris.add(new URI("http://127.0.0.1:8091/pools"));
115 | }
116 |
117 | if (prop.containsKey(COUCHBASE_BUCKET)) {
118 | bucket = prop.getProperty(COUCHBASE_BUCKET);
119 | }
120 |
121 | if (prop.containsKey(COUCHBASE_PASSWORD)) {
122 | password = prop.getProperty(COUCHBASE_PASSWORD);
123 | }
124 |
125 | if (prop.containsKey(SQL_CONN_STRING)) {
126 | sqlConnString = prop.getProperty(SQL_CONN_STRING);
127 | } else {
128 | throw new Exception(" JDBC Connection String not specified");
129 | }
130 |
131 | if (prop.containsKey(SQL_USER)) {
132 | sqlUser = prop.getProperty(SQL_USER);
133 | } else {
134 | throw new Exception(" JDBC User not specified");
135 | }
136 |
137 | if (prop.containsKey(SQL_PASSWORD)) {
138 | sqlPassword = prop.getProperty(SQL_PASSWORD);
139 | } else {
140 | throw new Exception(" JDBC Password not specified");
141 | }
142 |
143 | if (prop.containsKey(TABLES_LIST)) {
144 | tableList = prop.getProperty(TABLES_LIST).split(",");
145 | }
146 |
147 | if (prop.containsKey(CREATE_VIEWS)) {
148 | createTableViewEnable = Boolean.parseBoolean(prop.getProperty(CREATE_VIEWS));
149 | }
150 |
151 | if (prop.containsKey(TYPE_FIELD)) {
152 | typeField = prop.getProperty(TYPE_FIELD);
153 | }
154 |
155 | if (prop.containsKey(TYPE_CASE)) {
156 | typeFieldCase = prop.getProperty(TYPE_CASE);
157 | }
158 |
159 |
160 | System.out.println("\nImporting table(s)");
161 | System.out.println("\tfrom : \t" + sqlConnString);
162 | System.out.println("\tto : \t" + uris + " - " + bucket);
163 |
164 |
165 | } catch (Exception e) {
166 | System.out.println(e.getMessage() + "\n\n");
167 | System.exit(0);
168 | }
169 |
170 | }
171 |
172 | private void shutdown() throws SQLException {
173 | if (couchbaseClient != null) {
174 | couchbaseClient.shutdown(5, TimeUnit.SECONDS);
175 | }
176 | if (connection != null && !connection.isClosed()) {
177 | connection.close();
178 | }
179 |
180 | }
181 |
182 |
183 | public void SqlImporter() {
184 | }
185 |
186 |
187 | public void importData() throws Exception {
188 | if (tableList == null || tableList[0].equalsIgnoreCase("ALL")) {
189 | this.importAllTables();
190 | } else {
191 | for (int i = 0; i < tableList.length; i++) {
192 | this.importTable(tableList[i].trim());
193 | }
194 | }
195 |
196 | if (createTableViewEnable) {
197 | createTableViews();
198 | }
199 |
200 | }
201 |
202 |
203 | public void importAllTables() throws Exception {
204 | DatabaseMetaData md = this.getConnection().getMetaData();
205 | ResultSet rs = md.getTables(null, null, "%", null);
206 | while (rs.next()) {
207 | String tableName = rs.getString(3);
208 | importTable(tableName);
209 | }
210 | }
211 |
212 | public void importTable(String tableName) throws Exception {
213 | System.out.println("\n Exporting Table : " + tableName);
214 | String typeName = this.getNamewithCase(tableName, typeFieldCase);
215 | if (createTableViewEnable) {
216 | this.createViewsForPrimaryKey(tableName);
217 | }
218 | PreparedStatement preparedStatement = null;
219 | String selectSQL = "SELECT * FROM " + tableName;
220 | Gson gson = new Gson();
221 | try {
222 |
223 | preparedStatement = this.getConnection().prepareStatement(selectSQL);
224 | ResultSet rs = preparedStatement.executeQuery();
225 | ResultSetMetaData rsmd = rs.getMetaData();
226 | int numColumns = rsmd.getColumnCount();
227 |
228 | int numRow = 0;
229 | while (rs.next()) {
230 | Map map = new HashMap();
231 |
232 |
233 | for (int i = 1; i < numColumns + 1; i++) {
234 | String columnName = this.getNamewithCase(rsmd.getColumnName(i), typeFieldCase);
235 |
236 | if (rsmd.getColumnType(i) == java.sql.Types.ARRAY) {
237 | map.put(columnName, rs.getArray(columnName));
238 | } else if (rsmd.getColumnType(i) == java.sql.Types.BIGINT) {
239 | map.put(columnName, rs.getInt(columnName));
240 | } else if (rsmd.getColumnType(i) == java.sql.Types.BOOLEAN) {
241 | map.put(columnName, rs.getBoolean(columnName));
242 | } else if (rsmd.getColumnType(i) == java.sql.Types.BLOB) {
243 | map.put(columnName, rs.getBlob(columnName));
244 | } else if (rsmd.getColumnType(i) == java.sql.Types.DOUBLE) {
245 | map.put(columnName, rs.getDouble(columnName));
246 | } else if (rsmd.getColumnType(i) == java.sql.Types.FLOAT) {
247 | map.put(columnName, rs.getFloat(columnName));
248 | } else if (rsmd.getColumnType(i) == java.sql.Types.INTEGER) {
249 | map.put(columnName, rs.getInt(columnName));
250 | } else if (rsmd.getColumnType(i) == java.sql.Types.NVARCHAR) {
251 | map.put(columnName, rs.getNString(columnName));
252 | } else if (rsmd.getColumnType(i) == java.sql.Types.VARCHAR) {
253 | map.put(columnName, rs.getString(columnName));
254 | } else if (rsmd.getColumnType(i) == java.sql.Types.TINYINT) {
255 | map.put(columnName, rs.getInt(columnName));
256 | } else if (rsmd.getColumnType(i) == java.sql.Types.SMALLINT) {
257 | map.put(columnName, rs.getInt(columnName));
258 | } else if (rsmd.getColumnType(i) == java.sql.Types.DATE) {
259 | map.put(columnName, rs.getDate(columnName));
260 | } else if (rsmd.getColumnType(i) == java.sql.Types.TIMESTAMP) {
261 | map.put(columnName, rs.getTimestamp(columnName));
262 | } else {
263 | map.put(columnName, rs.getObject(columnName));
264 | }
265 | }
266 |
267 | if (typeField != null && ! typeField.isEmpty()) {
268 | map.put(typeField, typeName);
269 | }
270 |
271 | // use the rs number as key with table name
272 | this.getCouchbaseClient().set(typeName + ":" + rs.getRow(), gson.toJson(map)).get();
273 |
274 |
275 | numRow = rs.getRow();
276 | }
277 | System.out.println(" " + numRow + " records moved to Couchbase.");
278 |
279 | } catch (SQLException e) {
280 | System.out.println(e.getMessage());
281 | } finally {
282 | if (preparedStatement != null) {
283 | preparedStatement.close();
284 | }
285 | }
286 |
287 | }
288 |
289 | private void createViewsForPrimaryKey(String tableName) {
290 | String typeName = this.getNamewithCase(tableName, typeFieldCase);
291 | List pkCols = new ArrayList();
292 | DatabaseMetaData databaseMetaData = null;
293 | try {
294 | databaseMetaData = this.getConnection().getMetaData();
295 | ResultSet rs = databaseMetaData.getPrimaryKeys(null, null, tableName);
296 | while (rs.next()) {
297 | pkCols.add(rs.getString(4));
298 | }
299 | String[] array = pkCols.toArray(new String[pkCols.size()]);
300 |
301 | StringBuilder mapFunction = new StringBuilder();
302 | StringBuilder ifStatement = new StringBuilder();
303 | StringBuilder emitStatement = new StringBuilder();
304 |
305 |
306 | mapFunction.append("function (doc, meta) {\n")
307 | .append(" var idx = (meta.id).indexOf(\":\");\n")
308 | .append(" var docType = (meta.id).substring(0,idx); \n");
309 |
310 |
311 | if (array != null && array.length == 1) {
312 | ifStatement.append(" if (meta.type == 'json' && docType == '")
313 | .append(typeName)
314 | .append("' && doc.")
315 | .append( this.getNamewithCase( array[0], typeFieldCase) )
316 | .append(" ){ \n");
317 | emitStatement.append(" emit(doc.").append(array[0]).append(");");
318 | } else if (array != null && array.length > 1) {
319 | emitStatement.append(" emit([");
320 | ifStatement.append(" if (meta.type == 'json' && docType == '")
321 | .append(typeName)
322 | .append("' && ");
323 |
324 | for (int i = 0; i < array.length; i++) {
325 | emitStatement.append("doc.").append( this.getNamewithCase( array[i], typeFieldCase) );
326 | ifStatement.append("doc.").append( this.getNamewithCase( array[i], typeFieldCase) );
327 | if (i < (array.length-1)) {
328 | emitStatement.append(", ");
329 | ifStatement.append(" && ");
330 | }
331 | }
332 | ifStatement.append(" ){\n");
333 | emitStatement.append("]);\n");
334 | }
335 |
336 | mapFunction.append( ifStatement )
337 | .append(emitStatement)
338 | .append(" }\n")
339 | .append("}\n");
340 |
341 |
342 | System.out.println("\n\n Create Couchbase views for table "+ typeName);
343 | DesignDocument designDoc = new DesignDocument(tableName);
344 | String viewName = "by_pk";
345 | String reduceFunction = "_count";
346 | ViewDesign viewDesign = new ViewDesign(viewName, mapFunction.toString(), reduceFunction);
347 | designDoc.getViews().add(viewDesign);
348 | getCouchbaseClient().createDesignDoc(designDoc);
349 |
350 |
351 |
352 | } catch (SQLException e) {
353 | e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
354 | } catch (Exception e) {
355 | e.printStackTrace(); //To change body of catch statement use File | Settings | File Templates.
356 | }
357 | }
358 |
359 | private void createTableViews() throws Exception {
360 |
361 |
362 | System.out.println("\n\n Create Couchbase views for 'types' ....");
363 |
364 | DesignDocument designDoc = new DesignDocument("all");
365 |
366 | String viewName = "by_type";
367 | String mapFunction =
368 | "function (doc, meta) {\n" +
369 | " if (meta.type == \"json\") {\n" +
370 | " var idx = (meta.id).indexOf(\":\");\n" +
371 | " emit( (meta.id).substring(0,idx));\n" +
372 | " } \n" +
373 | "}";
374 | String reduceFunction = "_count";
375 | ViewDesign viewDesign = new ViewDesign(viewName, mapFunction, reduceFunction);
376 | designDoc.getViews().add(viewDesign);
377 | getCouchbaseClient().createDesignDoc(designDoc);
378 |
379 | }
380 |
381 |
382 | public CouchbaseClient getCouchbaseClient() throws Exception {
383 | if (couchbaseClient == null) {
384 | couchbaseClient = new CouchbaseClient(uris, bucket, password);
385 | }
386 | return couchbaseClient;
387 |
388 |
389 | }
390 |
391 | public void setCouchbaseClient(CouchbaseClient couchbaseClient) {
392 | this.couchbaseClient = couchbaseClient;
393 | }
394 |
395 | public Connection getConnection() throws SQLException {
396 | if (connection == null) {
397 | connection = DriverManager.getConnection(sqlConnString, sqlUser, sqlPassword);
398 | }
399 | return connection;
400 | }
401 |
402 | public void setConnection(Connection connection) {
403 | this.connection = connection;
404 | }
405 |
406 | private String getNamewithCase(String tablename, String nameType) {
407 | String returnValue = tablename;
408 | if (nameType.equalsIgnoreCase("lower")) {
409 | returnValue = tablename.toLowerCase();
410 | } else if (nameType.equalsIgnoreCase("upper")) {
411 | returnValue = tablename.toUpperCase();
412 | }
413 | return returnValue;
414 | }
415 | }
416 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
123 |
124 |
125 |
144 |
145 |
146 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
255 |
256 |
257 |
258 |
259 |
260 |
261 |
262 |
263 |
264 |
265 |
266 |
267 |
268 |
269 |
270 |
271 |
272 |
273 |
274 |
275 |
276 |
277 |
278 |
279 |
280 |
281 |
282 |
283 |
284 |
285 |
286 |
287 |
288 |
289 |
290 |
291 |
292 |
293 |
294 |
295 |
296 |
297 |
298 |
299 |
300 |
301 |
302 |
303 |
304 |
305 |
306 |
307 |
308 |
309 |
310 |
311 |
312 |
313 |
314 |
315 |
316 |
317 |
318 |
319 |
320 |
321 |
322 |
323 |
324 |
325 |
326 |
327 |
328 |
329 |
330 |
331 |
332 |
333 |
334 |
335 |
336 |
337 |
338 |
339 |
340 |
341 |
342 |
343 |
344 |
345 |
346 |
347 |
348 |
349 |
350 |
351 |
352 |
353 |
354 |
355 |
356 |
357 |
358 |
359 |
360 |
361 |
362 |
363 |
364 |
365 |
366 |
367 |
368 |
369 |
370 |
371 |
372 |
373 |
374 |
375 |
376 |
377 |
378 |
379 |
380 |
381 |
382 |
383 |
384 |
385 |
386 |
387 |
388 |
389 |
390 |
391 |
392 |
393 |
394 |
395 |
396 |
397 |
398 |
399 |
400 |
401 |
402 |
403 |
404 |
405 |
406 |
407 |
410 |
411 |
412 |
413 |
414 |
415 |
416 |
417 |
418 |
419 |
420 |
421 |
422 |
423 |
424 |
425 |
426 |
427 |
428 |
429 |
430 |
431 |
432 |
433 |
434 |
435 |
436 |
437 |
438 |
439 |
440 |
441 |
442 |
443 |
444 |
445 |
446 |
447 |
448 |
449 |
450 |
451 |
452 |
453 |
454 |
455 |
456 |
457 |
458 |
459 |
462 |
463 |
464 |
465 |
466 |
467 |
470 |
471 |
472 |
475 |
476 |
477 | localhost
478 | 5050
479 |
480 |
481 |
482 |
483 |
484 |
485 |
486 |
487 |
488 |
489 |
490 |
491 |
492 |
493 |
494 |
495 |
496 |
497 |
498 | false
499 |
500 |
501 |
502 |
503 | 1372480749500
504 | 1372480749500
505 |
506 |
507 |
508 |
509 |
510 |
511 |
512 |
513 |
514 |
515 |
516 |
517 |
518 |
519 |
520 |
521 |
522 |
523 |
524 |
525 |
526 |
527 |
528 |
529 |
530 |
531 |
532 |
533 |
534 |
535 |
536 |
537 |
538 |
539 |
540 |
541 |
542 |
543 |
544 |
545 |
553 |
554 |
555 |
556 |
557 |
558 |
559 |
560 |
561 |
562 |
563 |
564 |
565 |
566 |
567 |
568 |
569 |
570 |
571 |
572 |
573 |
574 |
575 |
576 |
577 |
578 |
579 |
580 |
581 |
582 |
583 |
584 |
585 |
586 |
587 |
588 |
589 |
590 |
591 |
592 |
593 |
594 |
595 |
596 |
597 |
598 |
599 |
600 |
601 |
602 |
603 |
604 |
605 |
606 |
607 |
608 |
609 |
610 |
611 |
612 |
613 |
614 |
615 |
616 |
617 |
618 |
619 |
620 |
621 |
622 |
623 |
624 |
625 |
626 |
627 |
628 |
629 |
630 |
631 |
632 |
633 |
634 |
635 |
636 |
637 |
638 |
639 |
640 |
641 |
642 |
643 |
644 |
645 |
646 |
647 |
648 |
649 |
650 |
651 |
652 |
653 |
654 |
655 |
656 |
657 |
658 |
659 |
660 |
661 |
662 |
663 |
664 |
665 |
666 |
667 |
668 |
669 |
670 |
671 |
672 |
673 |
674 |
675 |
676 |
677 |
678 |
679 |
680 |
681 |
682 |
683 |
684 |
685 |
686 |
687 |
688 |
689 |
690 |
691 |
692 |
693 |
694 |
695 |
696 |
697 |
698 |
699 |
700 |
701 |
702 |
703 |
704 |
705 |
706 |
707 |
708 |
709 |
710 |
711 |
712 |
713 |
714 |
715 |
716 |
717 |
722 |
723 |
724 |
725 |
726 |
727 | No facets are configured
728 |
729 |
734 |
735 |
736 |
737 |
738 |
739 |
740 |
745 |
746 |
747 |
748 |
749 |
750 | 1.6
751 |
752 |
757 |
758 |
759 |
760 |
761 |
762 | couchbase-sql-importer
763 |
764 |
770 |
771 |
772 |
773 |
774 |
775 | 1.6
776 |
777 |
782 |
783 |
784 |
785 |
786 |
787 | Maven: com.google.code.gson:gson:2.2.2
788 |
789 |
794 |
795 |
796 |
797 |
798 |
799 |
800 |
801 |
--------------------------------------------------------------------------------