├── .gitignore
├── .travis-jvmopts
├── .travis.yml
├── CHANGE_LOG.md
├── CLA.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE.txt
├── README.md
├── build.sbt
├── codecov.yml
├── logo.png
├── project
├── Dependencies.scala
├── build.properties
└── plugins.sbt
└── src
├── main
└── scala
│ └── com
│ └── agoda
│ └── kafka
│ └── connector
│ └── jdbc
│ ├── JdbcSourceConnector.scala
│ ├── JdbcSourceConnectorConfig.scala
│ ├── JdbcSourceConnectorConstants.scala
│ ├── JdbcSourceTask.scala
│ ├── JdbcSourceTaskConfig.scala
│ ├── models
│ ├── DatabaseProduct.scala
│ └── Mode.scala
│ ├── services
│ ├── DataService.scala
│ ├── IdBasedDataService.scala
│ ├── TimeBasedDataService.scala
│ └── TimeIdBasedDataService.scala
│ └── utils
│ ├── DataConverter.scala
│ └── Version.scala
└── test
└── scala
└── com
└── agoda
└── kafka
└── connector
└── jdbc
├── JdbcSourceConnectorConfigTest.scala
├── models
├── DatabaseProductTest.scala
└── ModeTest.scala
├── services
├── DataServiceTest.scala
├── IdBasedDataServiceTest.scala
├── TimeBasedDataServiceTest.scala
└── TimeIdBasedDataServiceTest.scala
└── utils
└── DataConverterTest.scala
/.gitignore:
--------------------------------------------------------------------------------
1 | # IDE specific
2 | .idea/
3 |
4 | # SBT specific
5 | target/
--------------------------------------------------------------------------------
/.travis-jvmopts:
--------------------------------------------------------------------------------
1 | # This is used to configure the sbt instance that Travis launches
2 |
3 | -Xms1G
4 | -Xmx1G
5 | -Xss2M
6 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: scala
2 | sudo: false
3 | scala:
4 | - "2.12.2"
5 | jdk:
6 | - oraclejdk8
7 | env:
8 | matrix:
9 | - SCRIPT="sbt -jvm-opts .travis-jvmopts clean coverage test coverageReport"
10 | # important to use eval, otherwise "&&" is passed as an argument to sbt rather than being processed by bash
11 | script: eval $SCRIPT
12 | cache:
13 | directories:
14 | - $HOME/.ivy2/cache
15 | - $HOME/.sbt/boot
16 | after_success:
17 | - bash <(curl -s https://codecov.io/bash) || echo 'Codecov failed to upload'
18 | notifications:
19 | webhooks:
20 | urls:
21 | - https://webhooks.gitter.im/e/1d93347f19b51af09514
22 | on_success: always # options: [always|never|change] default: always
23 | on_failure: always # options: [always|never|change] default: always
24 | on_start: never # options: [always|never|change] default: always
25 |
--------------------------------------------------------------------------------
/CHANGE_LOG.md:
--------------------------------------------------------------------------------
1 | Change Log
2 | ==========
3 |
4 | Release 1.2.0
5 | -------------
6 |
7 | ### Requirements
8 |
9 | * Scala 2.11.* / 2.12.*
10 | * Kafka 0.9.0.0
11 | * Kafka Connect 0.9.0.0
12 |
13 | ### Supported Databases
14 |
15 | * Mssql
16 | * Mysql
17 |
18 | ### Added
19 |
20 | * Extending support for MySQL database.
21 |
22 | Release 1.0.1
23 | -------------
24 |
25 | ### Fixed
26 |
27 | * NullPointerException in Kafka source connector while connecting to database.
28 |
29 | Release 1.0.0
30 | -------------
31 |
32 | ### Requirements
33 |
34 | * Scala 2.11.* / 2.12.*
35 | * Kafka 0.9.0.0
36 | * Kafka Connect 0.9.0.0
37 |
38 | ### Supported Databases
39 |
40 | * Mssql
41 |
42 | ### Added
43 |
44 | * Kafka source connector to capture change data using stored procedures.
--------------------------------------------------------------------------------
/CLA.md:
--------------------------------------------------------------------------------
1 | 
2 |
3 | # INDIVIDUAL CONTRIBUTOR LICENSE AGREEMENT
4 |
5 | Thank you for your interest in [Agoda.com](https://www.agoda.com) (the "Company") open source project. The Company asks you (“You” or “Contributor”) to sign this Contribution License Agreement in order to protect you as a Contributor and the Company and its users. This is not an assignment and it does not affect your ownership of your Contributions for any other purpose. It does, however, grant to the Company certain rights to your Contributions.
6 |
7 | Please complete, sign and return this Agreement to the Company below by using [GitHub](https://github.com)
8 |
9 | Please read this document carefully before signing and keep a copy for your records.
10 |
11 | - Full legal name:
12 | - Public name/Alias:
13 | - Mailing Address:
14 | - Location:
15 | - GitHub ID:
16 | - Telephone:
17 | - E-Mail:
18 | - Open Source Project(s):[Kafka JDBC Connector](https://github.com/agoda-com/kafka-jdbc-connector)
19 |
20 | You accept and agree to the following terms and conditions for Your past, present and future Contributions submitted to the Company.
21 |
22 | 1. Definitions.
23 |
24 | "You" (or "Your") shall mean the legal owner of the Contribution that is making this Agreement with the Company. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
25 |
26 | "Contribution" shall mean any original work of authorship, including any derivative works, modifications or additions to an existing work, that is intentionally submitted by You to the Company for inclusion in, or documentation of, any of the products owned or distributed by the Company (the "Work"). You acknowledge that the Company desires to have all contributions made by You under the terms of this CLA and, thus, this CLA will apply to all of your Contributions submitted both before and after the date of signature. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Company or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Company for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as: "Not a Contribution – Exempt from Contributor License Agreement dated _______.”
27 |
28 | 2. Grant of Copyright License.
29 |
30 | Subject to the terms and conditions of this Agreement, You hereby grant to the Company and to recipients of software distributed by the Company a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license, including a license, to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, distribute, or sell Your Contributions and the Work.
31 |
32 | 3. Grant of Patent License.
33 |
34 | Subject to the terms and conditions of this Agreement, You hereby grant to the Company and to recipients of the Work distributed by the Company a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable patent license to make, have made, use, offer to sell, sell, import, distribute, and otherwise transfer Your Contributions and the Work.
35 |
36 | 4. Ownership and third party rights.
37 |
38 | You represent that you are the sole legal owner and are legally entitled to grant the above license for the Contributions. If:
39 |
40 | - (A.) your employer(s) has intellectual property or other rights to your Contributions, you represent that you have both (i.) received express, prior, written permission to make Contributions on behalf of that employer; and (ii.) that your employer has waived any of its rights for or claims in your Contributions to the Company, or
41 |
42 | - (B.) if another individual or third party has rights to intellectual property to your Contributions – whether as a result of being a co-inventor, assignee, or other right, you represent that you have both (i.) received express, prior, written permission to make Contributions on behalf of that individual or third party; and (ii.) that such individual or third party has waived any of its rights for or claims in your Contributions to the Company.
43 |
44 | You will submit such written permission to the Company at the time of the submission of your Contribution.
45 |
46 | 5. Your original creation.
47 |
48 | You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include
49 | complete details (including required attributions and details of applicable license restrictions) of any third-party license or public domain licenses, or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware or should be aware and which are associated with any part of Your Contributions.
50 |
51 | 6. Support and warranties.
52 |
53 | You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, the Company acknowledges that You provide Your Contribution on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NONINFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE.
54 |
55 | 7. Third party owned creation(s).
56 |
57 | Should You wish to submit work that is not Your original creation, You may submit it to the Company separately from any Contribution, clearly identifying the complete details of its ownership and source, and any applicable license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work, for example: "Submitted on behalf of a third-party: [named here].” “Owned by third-party: [named here.]” or “Copyright held by third-party: [named here].”
58 |
59 | 8. Notification.
60 |
61 | You agree to notify the Company of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect, including if you become aware of any third party intellectual property rights that are infringed by your Contributions.
62 |
63 | 9. Assignment.
64 |
65 | Neither party may assign this Agreement without the other party’s consent which will not be unreasonably withheld; however, each party may assign this Agreement without the other party’s consent to an entity or individual that acquires all or substantially all of the business or assets of the assigning party or for an individual acquires all of the intellectual property rights in the Contribution owned by such individual, whether by merger, sale of assets, or otherwise, provided that such entity or individual assumes and agrees in writing to be bound by all of the obligations of the assigning party under this Agreement.
66 |
67 | 10. Applicable law and jurisdiction.
68 |
69 | The Agreement shall be governed by and construed and enforced in accordance with, the laws of the State of New York, without reference to conflicts of laws. All disputes arising out of or relating to this Agreement will be submitted to the exclusive jurisdiction of the state or federal courts of the State of New York, and each party irrevocably consents to such personal jurisdiction and waives all objections to this venue. The application of the United Nations Convention on the International Sale of Goods to the Agreement is disclaimed in its entirety.
70 |
71 | 11. Entire agreement.
72 |
73 | This Agreement is the entire agreement, both written or oral, with respect to the Contributions between the parties. No amendment, modification or waiver of any provision of this Agreement will be effective unless in writing and signed by both parties. If any provision of this Agreement is held to be invalid or unenforceable, the remaining portions will remain in full force and effect and such provision will be enforced to the maximum extent possible so as to affect the intent of the parties and will be reformed to the extent necessary to make such provision valid and enforceable. All notices and other communications herein permitted or required under this Agreement will be sent by postage prepaid, via registered or certified mail or overnight courier, return receipt requested, or delivered personally to the parties at their respective addresses, or to such other address as either party will give to the other party in the manner provided herein for giving notice. Notice will be considered given upon receipt.
74 |
75 |
76 |
77 | \___________________________________
78 |
79 | Contributor
80 |
81 | \___________________________________
82 |
83 | Date
84 |
85 | \___________________________________
86 |
87 | Print Name
88 |
89 |
90 |
91 | Acknowledged [Agoda.com](https://www.agoda.com)
92 |
93 | By: _________________________________
94 |
95 | Date: _______________________________
96 |
--------------------------------------------------------------------------------
/CODE_OF_CONDUCT.md:
--------------------------------------------------------------------------------
1 | # Contributor Covenant Code of Conduct
2 |
3 | ## Our Pledge
4 |
5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
6 |
7 | ## Our Standards
8 |
9 | Examples of behavior that contributes to creating a positive environment include:
10 |
11 | * Using welcoming and inclusive language
12 | * Being respectful of differing viewpoints and experiences
13 | * Gracefully accepting constructive criticism
14 | * Focusing on what is best for the community
15 | * Showing empathy towards other community members
16 |
17 | Examples of unacceptable behavior by participants include:
18 |
19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances
20 | * Trolling, insulting/derogatory comments, and personal or political attacks
21 | * Public or private harassment
22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission
23 | * Other conduct which could reasonably be considered inappropriate in a professional setting
24 |
25 | ## Our Responsibilities
26 |
27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
28 |
29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
30 |
31 | ## Scope
32 |
33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
34 |
35 | ## Enforcement
36 |
37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at arpan.chaudhury@agoda.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
38 |
39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
40 |
41 | ## Attribution
42 |
43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
44 |
45 | [homepage]: http://contributor-covenant.org
46 | [version]: http://contributor-covenant.org/version/1/4/
47 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | Contributing to Kafka JDBC Connector
2 | ------------------------------------
3 |
4 | In case you have questions about the contribution process or want to discuss about an issue please use the [kafka-connect-jdbc](https://gitter.im/kafka-jdbc-connector/Lobby) gitter channel.
5 |
6 | Tags
7 | ----
8 |
9 | * Enhancement - This tag is assigned to an issue when it addresses an enhancement to the existing code-base.
10 |
11 | * Duplicate - This tag is assigned to an issue when there is already an open ticket for the same issue.
12 |
13 | * Bug - This tag is assigned to an issue when it addresses a bug in existing code-base.
14 |
15 | * Invalid - This tag is assigned to an issue when it does not fall under the scope of this project.
16 |
17 | General Workflow
18 | ----------------
19 |
20 | The steps below describe how to get a patch into a master branch.
21 |
22 | 1. Find an open issue or create a new issue on [issue tracker](https://github.com/agoda-com/kafka-jdbc-connector/issues) for the work you want to contribute. Incase you wish to work on an existing issue make sure no one else is working on it. Comment on the issue declaring you are willing to take it up to avoid possible conflicts and reworks. Also avoid picking up issues tagged as Invalid.
23 | 2. [Fork the project](https://github.com/agoda-com/kafka-jdbc-connector#fork-destination-box) on GitHub. You'll need to create a feature-branch for your work on your fork, as this way you'll be able to submit a pull request against kafka-jdbc-connector.
24 | 3. Continue working on the feature branch until you are satisfied. Add tests for new features and modify existing tests if required.
25 | 4. Run all unit tests from sbt and make sure all of them pass.
26 | 5. Run code coverage to check if the lines of code you added are covered by unit tests.
27 | 6. Once your feature is complete, prepare the commit with appropriate message and the issue number. e.g. this commit is a sample #12.
28 | 7. Create a [pull request](https://help.github.com/articles/about-pull-requests/) and wait for the users to review.
29 | 8. Sign [Contributor License Agreement](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/CLA.md) if not already signed.
30 | 9. Once everything is said and done, your pull request gets merged. Your feature will be available with the next release.
31 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 |
2 | Apache License
3 | Version 2.0, January 2004
4 | http://www.apache.org/licenses/
5 |
6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7 |
8 | 1. Definitions.
9 |
10 | "License" shall mean the terms and conditions for use, reproduction,
11 | and distribution as defined by Sections 1 through 9 of this document.
12 |
13 | "Licensor" shall mean the copyright owner or entity authorized by
14 | the copyright owner that is granting the License.
15 |
16 | "Legal Entity" shall mean the union of the acting entity and all
17 | other entities that control, are controlled by, or are under common
18 | control with that entity. For the purposes of this definition,
19 | "control" means (i) the power, direct or indirect, to cause the
20 | direction or management of such entity, whether by contract or
21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
22 | outstanding shares, or (iii) beneficial ownership of such entity.
23 |
24 | "You" (or "Your") shall mean an individual or Legal Entity
25 | exercising permissions granted by this License.
26 |
27 | "Source" form shall mean the preferred form for making modifications,
28 | including but not limited to software source code, documentation
29 | source, and configuration files.
30 |
31 | "Object" form shall mean any form resulting from mechanical
32 | transformation or translation of a Source form, including but
33 | not limited to compiled object code, generated documentation,
34 | and conversions to other media types.
35 |
36 | "Work" shall mean the work of authorship, whether in Source or
37 | Object form, made available under the License, as indicated by a
38 | copyright notice that is included in or attached to the work
39 | (an example is provided in the Appendix below).
40 |
41 | "Derivative Works" shall mean any work, whether in Source or Object
42 | form, that is based on (or derived from) the Work and for which the
43 | editorial revisions, annotations, elaborations, or other modifications
44 | represent, as a whole, an original work of authorship. For the purposes
45 | of this License, Derivative Works shall not include works that remain
46 | separable from, or merely link (or bind by name) to the interfaces of,
47 | the Work and Derivative Works thereof.
48 |
49 | "Contribution" shall mean any work of authorship, including
50 | the original version of the Work and any modifications or additions
51 | to that Work or Derivative Works thereof, that is intentionally
52 | submitted to Licensor for inclusion in the Work by the copyright owner
53 | or by an individual or Legal Entity authorized to submit on behalf of
54 | the copyright owner. For the purposes of this definition, "submitted"
55 | means any form of electronic, verbal, or written communication sent
56 | to the Licensor or its representatives, including but not limited to
57 | communication on electronic mailing lists, source code control systems,
58 | and issue tracking systems that are managed by, or on behalf of, the
59 | Licensor for the purpose of discussing and improving the Work, but
60 | excluding communication that is conspicuously marked or otherwise
61 | designated in writing by the copyright owner as "Not a Contribution."
62 |
63 | "Contributor" shall mean Licensor and any individual or Legal Entity
64 | on behalf of whom a Contribution has been received by Licensor and
65 | subsequently incorporated within the Work.
66 |
67 | 2. Grant of Copyright License. Subject to the terms and conditions of
68 | this License, each Contributor hereby grants to You a perpetual,
69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70 | copyright license to reproduce, prepare Derivative Works of,
71 | publicly display, publicly perform, sublicense, and distribute the
72 | Work and such Derivative Works in Source or Object form.
73 |
74 | 3. Grant of Patent License. Subject to the terms and conditions of
75 | this License, each Contributor hereby grants to You a perpetual,
76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77 | (except as stated in this section) patent license to make, have made,
78 | use, offer to sell, sell, import, and otherwise transfer the Work,
79 | where such license applies only to those patent claims licensable
80 | by such Contributor that are necessarily infringed by their
81 | Contribution(s) alone or by combination of their Contribution(s)
82 | with the Work to which such Contribution(s) was submitted. If You
83 | institute patent litigation against any entity (including a
84 | cross-claim or counterclaim in a lawsuit) alleging that the Work
85 | or a Contribution incorporated within the Work constitutes direct
86 | or contributory patent infringement, then any patent licenses
87 | granted to You under this License for that Work shall terminate
88 | as of the date such litigation is filed.
89 |
90 | 4. Redistribution. You may reproduce and distribute copies of the
91 | Work or Derivative Works thereof in any medium, with or without
92 | modifications, and in Source or Object form, provided that You
93 | meet the following conditions:
94 |
95 | (a) You must give any other recipients of the Work or
96 | Derivative Works a copy of this License; and
97 |
98 | (b) You must cause any modified files to carry prominent notices
99 | stating that You changed the files; and
100 |
101 | (c) You must retain, in the Source form of any Derivative Works
102 | that You distribute, all copyright, patent, trademark, and
103 | attribution notices from the Source form of the Work,
104 | excluding those notices that do not pertain to any part of
105 | the Derivative Works; and
106 |
107 | (d) If the Work includes a "NOTICE" text file as part of its
108 | distribution, then any Derivative Works that You distribute must
109 | include a readable copy of the attribution notices contained
110 | within such NOTICE file, excluding those notices that do not
111 | pertain to any part of the Derivative Works, in at least one
112 | of the following places: within a NOTICE text file distributed
113 | as part of the Derivative Works; within the Source form or
114 | documentation, if provided along with the Derivative Works; or,
115 | within a display generated by the Derivative Works, if and
116 | wherever such third-party notices normally appear. The contents
117 | of the NOTICE file are for informational purposes only and
118 | do not modify the License. You may add Your own attribution
119 | notices within Derivative Works that You distribute, alongside
120 | or as an addendum to the NOTICE text from the Work, provided
121 | that such additional attribution notices cannot be construed
122 | as modifying the License.
123 |
124 | You may add Your own copyright statement to Your modifications and
125 | may provide additional or different license terms and conditions
126 | for use, reproduction, or distribution of Your modifications, or
127 | for any such Derivative Works as a whole, provided Your use,
128 | reproduction, and distribution of the Work otherwise complies with
129 | the conditions stated in this License.
130 |
131 | 5. Submission of Contributions. Unless You explicitly state otherwise,
132 | any Contribution intentionally submitted for inclusion in the Work
133 | by You to the Licensor shall be under the terms and conditions of
134 | this License, without any additional terms or conditions.
135 | Notwithstanding the above, nothing herein shall supersede or modify
136 | the terms of any separate license agreement you may have executed
137 | with Licensor regarding such Contributions.
138 |
139 | 6. Trademarks. This License does not grant permission to use the trade
140 | names, trademarks, service marks, or product names of the Licensor,
141 | except as required for reasonable and customary use in describing the
142 | origin of the Work and reproducing the content of the NOTICE file.
143 |
144 | 7. Disclaimer of Warranty. Unless required by applicable law or
145 | agreed to in writing, Licensor provides the Work (and each
146 | Contributor provides its Contributions) on an "AS IS" BASIS,
147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 | implied, including, without limitation, any warranties or conditions
149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 | PARTICULAR PURPOSE. You are solely responsible for determining the
151 | appropriateness of using or redistributing the Work and assume any
152 | risks associated with Your exercise of permissions under this License.
153 |
154 | 8. Limitation of Liability. In no event and under no legal theory,
155 | whether in tort (including negligence), contract, or otherwise,
156 | unless required by applicable law (such as deliberate and grossly
157 | negligent acts) or agreed to in writing, shall any Contributor be
158 | liable to You for damages, including any direct, indirect, special,
159 | incidental, or consequential damages of any character arising as a
160 | result of this License or out of the use or inability to use the
161 | Work (including but not limited to damages for loss of goodwill,
162 | work stoppage, computer failure or malfunction, or any and all
163 | other commercial damages or losses), even if such Contributor
164 | has been advised of the possibility of such damages.
165 |
166 | 9. Accepting Warranty or Additional Liability. While redistributing
167 | the Work or Derivative Works thereof, You may choose to offer,
168 | and charge a fee for, acceptance of support, warranty, indemnity,
169 | or other liability obligations and/or rights consistent with this
170 | License. However, in accepting such obligations, You may act only
171 | on Your own behalf and on Your sole responsibility, not on behalf
172 | of any other Contributor, and only if You agree to indemnify,
173 | defend, and hold each Contributor harmless for any liability
174 | incurred by, or claims asserted against, such Contributor by reason
175 | of your accepting any such warranty or additional liability.
176 |
177 | END OF TERMS AND CONDITIONS
178 |
179 | APPENDIX: How to apply the Apache License to your work.
180 |
181 | To apply the Apache License to your work, attach the following
182 | boilerplate notice, with the fields enclosed by brackets "[]"
183 | replaced with your own identifying information. (Don't include
184 | the brackets!) The text should be enclosed in the appropriate
185 | comment syntax for the file format. We also recommend that a
186 | file or class name and description of purpose be included on the
187 | same "printed page" as the copyright notice for easier
188 | identification within third-party archives.
189 |
190 | Copyright [yyyy] [name of copyright owner]
191 |
192 | Licensed under the Apache License, Version 2.0 (the "License");
193 | you may not use this file except in compliance with the License.
194 | You may obtain a copy of the License at
195 |
196 | http://www.apache.org/licenses/LICENSE-2.0
197 |
198 | Unless required by applicable law or agreed to in writing, software
199 | distributed under the License is distributed on an "AS IS" BASIS,
200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 | See the License for the specific language governing permissions and
202 | limitations under the License.
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | [](https://travis-ci.org/agoda-com/kafka-jdbc-connector)
2 | [](https://gitter.im/kafka-jdbc-connector/Lobby)
3 | [](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/LICENSE.txt)
4 | [](https://maven-badges.herokuapp.com/maven-central/com.agoda/kafka-jdbc-connector_2.11)
5 | [](https://github.com/agoda-com/kafka-jdbc-connector)
6 | [](https://cla-assistant.io/agoda-com/kafka-jdbc-connector)
7 |
8 | Kafka JDBC Connector
9 | ====================
10 |
11 | *Simple way to copy data from relational databases into kafka.*
12 |
13 | To copy data between Kafka and another system, users create a Connector for the system which they want to pull data from or push data to. Connectors come in two flavors: *SourceConnectors* to import data from another system and *SinkConnectors* to export data from Kafka to other datasources. This project is an implementation of *SourceConnector* which allows users to copy data from relational databases into Kafka topics. It provides a flexible way to keep logic of selecting next batch of records inside database stored procedures which is invoked from the connector tasks in each poll cycle.
14 |
15 | Following are few advantages of using stored procedures
16 |
17 | * Select only required columns to be returned as a record.
18 | * Filtering / Grouping rows returned to connector with SQL.
19 | * Changing logic of returned batch without reconfiguring connector.
20 |
21 | Install
22 | -------
23 |
24 | build.sbt
25 |
26 | ```scala
27 | libraryDependencies ++= Seq("com.agoda" %% "kafka-jdbc-connector" % "1.2.0")
28 | ```
29 |
30 | Change Log
31 | ----------
32 |
33 | Please refer to [Change Log](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/CHANGE_LOG.md) for requirements, supported databases and project changes.
34 |
35 | Examples
36 | --------
37 |
38 | Click [here](https://github.com/agoda-com/kafka-jdbc-connector-samples) for examples.
39 |
40 | ### Timestamp mode
41 |
42 | Create a stored procedure in MSSQL database
43 |
44 | ```
45 | create procedure [dbo].[cdc_table]
46 | @time datetime,
47 | @batch int
48 | as
49 | begin
50 | select top (@batch) *
51 | from cdc.table_ct as a
52 | left join cdc.lsn_time_mapping as b
53 | on a._$start_lsn = b.start_lsn
54 | where b.tran_end_time > @time
55 | order by b.tran_end_time asc
56 | end
57 | ```
58 |
59 | Post the following configutation to Kafka Connect rest interface
60 |
61 | ```
62 | {
63 | "name" : "cdc_timestamp",
64 | "config" : {
65 | "tasks.max": "1",
66 | "connector.class": "com.agoda.kafka.connector.jdbc.JdbcSourceConnector",
67 | "connection.url" : "jdbc:sqlserver://localhost:1433;user=sa;password=Passw0rd",
68 | "mode" : "timestamp",
69 | "stored-procedure.name" : "cdc_table",
70 | "topic" : "cdc-table-changelogs",
71 | "batch.max.rows.variable.name" : "batch",
72 | "timestamp.variable.name" : "time",
73 | "timestamp.field.name" : "tran_end_time"
74 | }
75 | }
76 | ```
77 |
78 | ### Incrementing mode
79 |
80 | Create a stored procedure in MSSQL database
81 |
82 | ```
83 | create procedure [dbo].[cdc_table]
84 | @id int,
85 | @batch int
86 | as
87 | begin
88 | select top (@batch) *
89 | from cdc.table_ct
90 | where auto_incrementing_id > @id
91 | order by auto_incrementing_id asc
92 | end
93 | ```
94 |
95 | Post the following configutation to Kafka Connect rest interface
96 |
97 | ```
98 | {
99 | "name" : "cdc_incrementing",
100 | "config" : {
101 | "tasks.max": "1",
102 | "connector.class": "com.agoda.kafka.connector.jdbc.JdbcSourceConnector",
103 | "connection.url" : "jdbc:sqlserver://localhost:1433;user=sa;password=Passw0rd",
104 | "mode" : "incrementing",
105 | "stored-procedure.name" : "cdc_table",
106 | "topic" : "cdc-table-changelogs",
107 | "batch.max.rows.variable.name" : "batch",
108 | "incrementing.variable.name" : "id",
109 | "incrementing.field.name" : "auto_incrementing_id"
110 | }
111 | }
112 | ```
113 |
114 | ### Timestamp + Incrementing mode
115 |
116 | Create a stored procedure in MSSQL database
117 |
118 | ```
119 | create procedure [dbo].[cdc_table]
120 | @time datetime,
121 | @id int,
122 | @batch int
123 | as
124 | begin
125 | select top (@batch) *
126 | from cdc.table_ct as a
127 | left join cdc.lsn_time_mapping as b
128 | on a._$start_lsn = b.start_lsn
129 | where b.tran_end_time > @time
130 | and a.auto_incrementing_id > @id
131 | order by b.tran_end_time, a.auto_incrementing_id asc
132 | end
133 | ```
134 |
135 | Post the following configutation to Kafka Connect rest interface
136 |
137 | ```
138 | {
139 | "name" : "cdc_timestamp_incrementing",
140 | "config" : {
141 | "tasks.max": "1",
142 | "connector.class": "com.agoda.kafka.connector.jdbc.JdbcSourceConnector",
143 | "connection.url" : "jdbc:sqlserver://localhost:1433;user=sa;password=Passw0rd",
144 | "mode" : "timestamp+incrementing",
145 | "stored-procedure.name" : "cdc_table",
146 | "topic" : "cdc-table-changelogs",
147 | "batch.max.rows.variable.name" : "batch",
148 | "timestamp.variable.name" : "time",
149 | "timestamp.field.name" : "tran_end_time",
150 | "incrementing.variable.name" : "id",
151 | "incrementing.field.name" : "auto_incrementing_id"
152 | }
153 | }
154 | ```
155 |
156 | Contributing
157 | ------------
158 |
159 | **Kafka JDBC Connector** is an open source project, and depends on its users to improve it. We are more than happy to find you interested in taking the project forward.
160 |
161 | Kindly refer to the [Contribution Guidelines](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/CONTRIBUTING.md) for detailed information.
162 |
163 | Code of Conduct
164 | ---------------
165 |
166 | Please refer to [Code of Conduct](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/CODE_OF_CONDUCT.md) document.
167 |
168 | License
169 | -------
170 |
171 | Kafka JDBC Connector is Open Source and available under the [Apache License, Version 2.0](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/LICENSE.txt).
172 |
--------------------------------------------------------------------------------
/build.sbt:
--------------------------------------------------------------------------------
1 | import sbt._
2 | import sbt.Keys._
3 |
4 | lazy val `kafka-jdbc-connector` =
5 | (project in file("."))
6 | .settings(
7 | name := "kafka-jdbc-connector",
8 | version := "1.2.0",
9 | organization := "com.agoda",
10 | scalaVersion := "2.11.7",
11 | crossScalaVersions := Seq("2.11.7", "2.12.2"),
12 | libraryDependencies ++= Dependencies.Compile.kafkaJdbcConnector ++ Dependencies.Test.kafkaJdbcConnector,
13 | fork in Test := true
14 | )
15 | .enablePlugins(BuildInfoPlugin)
16 | .settings(
17 | buildInfoKeys := Seq[BuildInfoKey](version),
18 | buildInfoPackage := organization.value
19 | )
20 | .settings(
21 | test in assembly := {},
22 | assemblyJarName in assembly := s"kafka-jdbc-connector-${version.value}.jar"
23 | )
24 | .settings(
25 | useGpg := true,
26 | pgpPublicRing := file("~/.sbt/gpg/pubring.asc"),
27 | pgpSecretRing := file("~/.sbt/gpg/secring.asc"),
28 | publishTo := Some(
29 | if (isSnapshot.value) Opts.resolver.sonatypeSnapshots
30 | else Opts.resolver.sonatypeStaging
31 | ),
32 | credentials += Credentials(Path.userHome / ".ivy2" / ".credentials"),
33 | publishMavenStyle := true,
34 | licenses := Seq("Apache 2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0")),
35 | homepage := Some(url("https://github.com/agoda-com/kafka-jdbc-connector")),
36 | scmInfo := Some(
37 | ScmInfo(
38 | url("https://github.com/agoda-com/kafka-jdbc-connector"),
39 | "scm:git@github.com:agoda-com/kafka-jdbc-connector.git"
40 | )
41 | ),
42 | developers := List(
43 | Developer(
44 | id="arpanchaudhury",
45 | name="Arpan Chaudhury",
46 | email="arpan.chaudhury@agoda.com",
47 | url=url("https://github.com/arpanchaudhury")
48 | )
49 | )
50 | )
51 | .settings(
52 | coverageExcludedPackages := Seq(
53 | "com.agoda.BuildInfo",
54 | "com.agoda.kafka.connector.jdbc.JdbcSourceConnector",
55 | "com.agoda.kafka.connector.jdbc.JdbcSourceTask",
56 | "com.agoda.kafka.connector.jdbc.utils.Version"
57 | ).mkString(";")
58 | )
59 |
--------------------------------------------------------------------------------
/codecov.yml:
--------------------------------------------------------------------------------
1 | ignore:
2 | - "src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnector.scala"
3 | - "src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceTask.scala"
--------------------------------------------------------------------------------
/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/agoda-com/kafka-jdbc-connector/280ae243ec2c6857f18f0ec2d8717d18b88914c7/logo.png
--------------------------------------------------------------------------------
/project/Dependencies.scala:
--------------------------------------------------------------------------------
1 | import sbt._
2 |
3 | object Dependencies {
4 | private val ScalaTestV = "3.0.1"
5 |
6 | private val LogBack = "ch.qos.logback" % "logback-classic" % "1.2.3"
7 | private val ScalaLogging = "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0"
8 | private val KafkaConnectApi = "org.apache.kafka" % "connect-api" % "0.9.0.0"
9 | private val Enumeratum = "com.beachape" %% "enumeratum" % "1.5.12"
10 | private val Scalatics = "org.scalactic" %% "scalactic" % ScalaTestV % "test"
11 | private val ScalaTest = "org.scalatest" %% "scalatest" % ScalaTestV % "test"
12 | private val Mockito = "org.mockito" % "mockito-core" % "2.10.0" % "test"
13 |
14 | object Compile {
15 | def kafkaJdbcConnector = Seq(LogBack, ScalaLogging, KafkaConnectApi, Enumeratum)
16 | }
17 |
18 | object Test {
19 | def kafkaJdbcConnector = Seq(LogBack, ScalaLogging, Scalatics, ScalaTest, Mockito)
20 | }
21 | }
--------------------------------------------------------------------------------
/project/build.properties:
--------------------------------------------------------------------------------
1 | sbt.version = 0.13.13
--------------------------------------------------------------------------------
/project/plugins.sbt:
--------------------------------------------------------------------------------
1 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.6.1")
2 |
3 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4")
4 |
5 | addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2")
6 |
7 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1")
8 |
9 | addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0")
10 |
11 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0")
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnector.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc
2 |
3 | import java.util
4 |
5 | import com.agoda.kafka.connector.jdbc.utils.Version
6 | import org.apache.kafka.connect.errors.ConnectException
7 | import org.apache.kafka.connect.source.{SourceConnector, SourceTask}
8 | import org.slf4j.LoggerFactory
9 |
10 | import scala.collection.JavaConverters._
11 | import scala.util.{Failure, Success, Try}
12 |
13 | class JdbcSourceConnector extends SourceConnector {
14 | private val logger = LoggerFactory.getLogger(classOf[JdbcSourceConnector])
15 |
16 | private var config: JdbcSourceConnectorConfig = _
17 |
18 | /**
19 | * @return version of this connector
20 | */
21 | override def version: String = Version.getVersion
22 |
23 | /**
24 | * invoked by kafka-connect runtime to start this connector
25 | *
26 | * @param props properties required to start this connector
27 | */
28 | override def start(props: util.Map[String, String]): Unit = {
29 | Try (new JdbcSourceConnectorConfig(props.asScala.toMap)) match {
30 | case Success(c) => config = c
31 | case Failure(e) => logger.error("Couldn't start com.agoda.kafka.connector.jdbc.JdbcSourceConnector due to configuration error", new ConnectException(e))
32 | }
33 | }
34 |
35 | /**
36 | * invoked by kafka-connect runtime to stop this connector
37 | */
38 | override def stop(): Unit = {
39 | logger.debug("Stopping kafka source connector")
40 | }
41 |
42 | /**
43 | * invoked by kafka-connect runtime to instantiate SourceTask which polls data from external data store and saves into kafka
44 | *
45 | * @return class of source task to be created
46 | */
47 | override def taskClass(): Class[_ <: SourceTask] = classOf[JdbcSourceTask]
48 |
49 | /**
50 | * returns a set of configurations for tasks based on the current configuration
51 | *
52 | * @param maxTasks maximum number of configurations to generate
53 | * @return configurations for tasks
54 | */
55 | override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = List(config.properties.asJava).asJava
56 | }
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnectorConfig.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc
2 |
3 | import java.sql.Timestamp
4 | import java.util.TimeZone
5 |
6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants._
7 | import com.agoda.kafka.connector.jdbc.models.Mode
8 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampIncrementingMode, TimestampMode}
9 |
10 | /**
11 | * @constructor
12 | * @param properties is set of configurations required to create JdbcSourceConnectorConfig
13 | */
14 | class JdbcSourceConnectorConfig(val properties: Map[String, String]) {
15 |
16 | require(
17 | properties.contains(CONNECTION_URL_CONFIG) &&
18 | properties.contains(MODE_CONFIG) &&
19 | properties.contains(STORED_PROCEDURE_NAME_CONFIG) &&
20 | properties.contains(TOPIC_CONFIG) &&
21 | properties.contains(BATCH_MAX_ROWS_VARIABLE_NAME_CONFIG),
22 | s"""Required connector properties:
23 | | $CONNECTION_URL_CONFIG
24 | | $MODE_CONFIG
25 | | $STORED_PROCEDURE_NAME_CONFIG
26 | | $TOPIC_CONFIG
27 | | $BATCH_MAX_ROWS_VARIABLE_NAME_CONFIG""".stripMargin
28 | )
29 |
30 | require(
31 | Mode.withName(properties(MODE_CONFIG)) match {
32 | case TimestampMode => properties.contains(TIMESTAMP_VARIABLE_NAME_CONFIG) &&
33 | properties.contains(TIMESTAMP_FIELD_NAME_CONFIG)
34 | case IncrementingMode => properties.contains(INCREMENTING_VARIABLE_NAME_CONFIG) &&
35 | properties.contains(INCREMENTING_FIELD_NAME_CONFIG)
36 | case TimestampIncrementingMode => properties.contains(TIMESTAMP_VARIABLE_NAME_CONFIG) &&
37 | properties.contains(TIMESTAMP_FIELD_NAME_CONFIG) &&
38 | properties.contains(INCREMENTING_VARIABLE_NAME_CONFIG) &&
39 | properties.contains(INCREMENTING_FIELD_NAME_CONFIG)
40 | },
41 | Mode.withName(properties(MODE_CONFIG)) match {
42 | case TimestampMode => s"""Required connector properties:
43 | | $TIMESTAMP_VARIABLE_NAME_CONFIG
44 | | $TIMESTAMP_FIELD_NAME_CONFIG""".stripMargin
45 | case IncrementingMode => s"""Required connector properties:
46 | | $INCREMENTING_VARIABLE_NAME_CONFIG
47 | | $INCREMENTING_FIELD_NAME_CONFIG""".stripMargin
48 | case TimestampIncrementingMode => s"""Required connector properties:
49 | | $TIMESTAMP_VARIABLE_NAME_CONFIG
50 | | $TIMESTAMP_FIELD_NAME_CONFIG
51 | | $INCREMENTING_VARIABLE_NAME_CONFIG
52 | | $INCREMENTING_FIELD_NAME_CONFIG""".stripMargin
53 | }
54 | )
55 |
56 | /**
57 | * @return database connection url
58 | */
59 | def getConnectionUrl: String = properties(CONNECTION_URL_CONFIG)
60 |
61 | /**
62 | * @return mode of operation [[IncrementingMode]], [[TimestampMode]], [[TimestampIncrementingMode]]
63 | */
64 | def getMode: Mode = Mode.withName(properties(MODE_CONFIG))
65 |
66 | /**
67 | * @return stored procedure name
68 | */
69 | def getStoredProcedureName: String = properties(STORED_PROCEDURE_NAME_CONFIG)
70 |
71 | /**
72 | * @return kafka topic name
73 | */
74 | def getTopic: String = properties(TOPIC_CONFIG)
75 |
76 | /**
77 | * @return database poll interval
78 | */
79 | def getPollInterval: Long = properties.getOrElse(POLL_INTERVAL_MS_CONFIG, POLL_INTERVAL_MS_DEFAULT).toLong
80 |
81 | /**
82 | * @return number of records fetched in each poll
83 | */
84 | def getMaxBatchSize: Int = properties.getOrElse(BATCH_MAX_ROWS_CONFIG, BATCH_MAX_ROWS_DEFAULT).toInt
85 |
86 | /**
87 | * @return batch size variable name in stored procedure
88 | */
89 | def getMaxBatchSizeVariableName: String = properties(BATCH_MAX_ROWS_VARIABLE_NAME_CONFIG)
90 |
91 | /**
92 | * @return timestamp offset variable name in stored procedure
93 | */
94 | def getTimestampVariableName: Option[String] = properties.get(TIMESTAMP_VARIABLE_NAME_CONFIG)
95 |
96 | /**
97 | * @return timestamp offset field name in record
98 | */
99 | def getTimestampFieldName: Option[String] = properties.get(TIMESTAMP_FIELD_NAME_CONFIG)
100 |
101 | /**
102 | * @return incrementing offset variable name in stored procedure
103 | */
104 | def getIncrementingVariableName: Option[String] = properties.get(INCREMENTING_VARIABLE_NAME_CONFIG)
105 |
106 | /**
107 | * @return incrementing offset field name in record
108 | */
109 | def getIncrementingFieldName: Option[String] = properties.get(INCREMENTING_FIELD_NAME_CONFIG)
110 |
111 | /**
112 | * @return initial timestamp offset
113 | */
114 | def getTimestampOffset: Long = {
115 | properties
116 | .get(TIMESTAMP_OFFSET_CONFIG)
117 | .map(o => new Timestamp(Timestamp.valueOf(o).getTime + TimeZone.getDefault.getRawOffset))
118 | .getOrElse(TIMESTAMP_OFFSET_DEFAULT).getTime
119 | }
120 |
121 | /**
122 | * @return initial incrementing offset
123 | */
124 | def getIncrementingOffset: Long = properties.getOrElse(INCREMENTING_OFFSET_CONFIG, INCREMENTING_OFFSET_DEFAULT).toLong
125 |
126 | /**
127 | * @return optional field name to be used as kafka message key
128 | */
129 | def getKeyField: Option[String] = properties.get(KEY_FIELD_NAME_CONFIG)
130 | }
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnectorConstants.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc
2 |
3 | import java.sql.Timestamp
4 |
5 | object JdbcSourceConnectorConstants {
6 |
7 | val STORED_PROCEDURE_NAME_KEY = "stored-procedure.name"
8 |
9 | val CONNECTION_URL_CONFIG = "connection.url"
10 |
11 | val MODE_CONFIG = "mode"
12 | val TIMESTAMP_VARIABLE_NAME_CONFIG = "timestamp.variable.name"
13 | val TIMESTAMP_FIELD_NAME_CONFIG = "timestamp.field.name"
14 | val INCREMENTING_VARIABLE_NAME_CONFIG = "incrementing.variable.name"
15 | val INCREMENTING_FIELD_NAME_CONFIG = "incrementing.field.name"
16 |
17 | val STORED_PROCEDURE_NAME_CONFIG = "stored-procedure.name"
18 |
19 | val TOPIC_CONFIG = "topic"
20 |
21 | val POLL_INTERVAL_MS_CONFIG = "poll.interval.ms"
22 | val POLL_INTERVAL_MS_DEFAULT = "5000"
23 |
24 | val BATCH_MAX_ROWS_VARIABLE_NAME_CONFIG = "batch.max.rows.variable.name"
25 | val BATCH_MAX_ROWS_CONFIG = "batch.max.records"
26 | val BATCH_MAX_ROWS_DEFAULT = "100"
27 |
28 | val TIMESTAMP_OFFSET_CONFIG = "timestamp.offset"
29 | val TIMESTAMP_OFFSET_DEFAULT = new Timestamp(0L)
30 | val INCREMENTING_OFFSET_CONFIG = "incrementing.offset"
31 | val INCREMENTING_OFFSET_DEFAULT = "0"
32 |
33 | val KEY_FIELD_NAME_CONFIG = "key.field.name"
34 |
35 | val TASKS_MAX_CONFIG = "tasks.max"
36 | val CONNECTOR_CLASS = "connector.class"
37 | }
38 |
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceTask.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc
2 |
3 | import java.sql.{Connection, DriverManager, SQLException}
4 | import java.util
5 | import java.util.concurrent.atomic.AtomicBoolean
6 |
7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct
8 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampIncrementingMode, TimestampMode}
9 | import com.agoda.kafka.connector.jdbc.services.{DataService, IdBasedDataService, TimeBasedDataService, TimeIdBasedDataService}
10 | import com.agoda.kafka.connector.jdbc.utils.{DataConverter, Version}
11 | import org.apache.kafka.connect.errors.ConnectException
12 | import org.apache.kafka.connect.source.{SourceRecord, SourceTask}
13 | import org.slf4j.LoggerFactory
14 |
15 | import scala.collection.JavaConverters._
16 | import scala.concurrent.duration._
17 | import scala.util.{Failure, Success, Try}
18 |
19 | class JdbcSourceTask extends SourceTask {
20 | private val logger = LoggerFactory.getLogger(classOf[JdbcSourceTask])
21 | private val dataConverter = new DataConverter
22 |
23 | private var config: JdbcSourceTaskConfig = _
24 | private var db: Connection = _
25 | private var dataService: DataService = _
26 | private var running: AtomicBoolean = _
27 |
28 | override def version(): String = Version.getVersion
29 |
30 | /**
31 | * invoked by kafka-connect runtime to start this task
32 | *
33 | * @param props properties required to start this task
34 | */
35 | override def start(props: util.Map[String, String]): Unit = {
36 | Try(new JdbcSourceTaskConfig(props.asScala.toMap)) match {
37 | case Success(c) => config = c
38 | case Failure(e) => logger.error("Couldn't start com.agoda.kafka.connector.jdbc.JdbcSourceTask due to configuration error", new ConnectException(e))
39 | }
40 |
41 | val dbUrl = config.getConnectionUrl
42 | logger.debug(s"Trying to connect to $dbUrl")
43 | Try(DriverManager.getConnection(dbUrl)) match {
44 | case Success(c) => db = c
45 | case Failure(e: SQLException) => logger.error(s"Couldn't open connection to $dbUrl : ", e)
46 | throw new ConnectException(e)
47 | case Failure(e) => logger.error(s"Couldn't open connection to $dbUrl : ", e)
48 | throw e
49 | }
50 |
51 | val databaseProduct = DatabaseProduct.withName(db.getMetaData.getDatabaseProductName)
52 |
53 | val offset = context.offsetStorageReader().offset(
54 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> config.getStoredProcedureName).asJava
55 | )
56 |
57 | val storedProcedureName = config.getStoredProcedureName
58 | val timestampVariableNameOpt = config.getTimestampVariableName
59 | val timestampFieldNameOpt = config.getTimestampFieldName
60 | val incrementingVariableNameOpt = config.getIncrementingVariableName
61 | val incrementingFieldNameOpt = config.getIncrementingFieldName
62 | val batchSize = config.getMaxBatchSize
63 | val batchSizeVariableName = config.getMaxBatchSizeVariableName
64 | val topic = config.getTopic
65 | val keyFieldOpt = config.getKeyField
66 |
67 | config.getMode match {
68 | case TimestampMode =>
69 | val timestampOffset = Try(offset.get(TimestampMode.entryName)).map(_.toString.toLong).getOrElse(config.getTimestampOffset)
70 | dataService = TimeBasedDataService(databaseProduct, storedProcedureName, batchSize, batchSizeVariableName,
71 | timestampVariableNameOpt.get, timestampOffset, timestampFieldNameOpt.get, topic, keyFieldOpt, dataConverter)
72 |
73 | case IncrementingMode =>
74 | val incrementingOffset = Try(offset.get(IncrementingMode.entryName)).map(_.toString.toLong).getOrElse(config.getIncrementingOffset)
75 | dataService = IdBasedDataService(databaseProduct, storedProcedureName, batchSize, batchSizeVariableName,
76 | incrementingVariableNameOpt.get, incrementingOffset, incrementingFieldNameOpt.get, topic, keyFieldOpt, dataConverter)
77 |
78 | case TimestampIncrementingMode =>
79 | val timestampOffset = Try(offset.get(TimestampMode.entryName)).map(_.toString.toLong).getOrElse(config.getTimestampOffset)
80 | val incrementingOffset = Try(offset.get(IncrementingMode.entryName)).map(_.toString.toLong).getOrElse(config.getIncrementingOffset)
81 | dataService = TimeIdBasedDataService(databaseProduct, storedProcedureName, batchSize, batchSizeVariableName,
82 | timestampVariableNameOpt.get, timestampOffset, incrementingVariableNameOpt.get, incrementingOffset,
83 | timestampFieldNameOpt.get, incrementingFieldNameOpt.get, topic, keyFieldOpt, dataConverter)
84 | }
85 |
86 | running = new AtomicBoolean(true)
87 | }
88 |
89 | /**
90 | * invoked by kafka-connect runtime to stop this task
91 | */
92 | override def stop(): Unit = {
93 | if (running != null) running.set(false)
94 | if (db != null) {
95 | logger.debug("Trying to close database connection")
96 | Try(db.close()) match {
97 | case Success(_) =>
98 | case Failure(e) => logger.error("Failed to close database connection: ", e)
99 | }
100 | }
101 | }
102 |
103 | /**
104 | * invoked by kafka-connect runtime to poll data in [[JdbcSourceConnectorConstants.POLL_INTERVAL_MS_CONFIG]] interval
105 | */
106 | override def poll(): util.List[SourceRecord] = this.synchronized { if(running.get) fetchRecords else null }
107 |
108 | private def fetchRecords: util.List[SourceRecord] = {
109 | logger.debug("Polling new data ...")
110 | val pollInterval = config.getPollInterval
111 | val startTime = System.currentTimeMillis
112 | val fetchedRecords = dataService.getRecords(db, pollInterval.millis) match {
113 | case Success(records) => if(records.isEmpty) logger.info(s"No updates for $dataService")
114 | else logger.info(s"Returning ${records.size} records for $dataService")
115 | records
116 | case Failure(e: SQLException) => logger.error(s"Failed to fetch data for $dataService: ", e)
117 | Seq.empty[SourceRecord]
118 | case Failure(e: Throwable) => logger.error(s"Failed to fetch data for $dataService: ", e)
119 | Seq.empty[SourceRecord]
120 | }
121 | val endTime = System.currentTimeMillis
122 | val elapsedTime = endTime - startTime
123 |
124 | if(elapsedTime < pollInterval) Thread.sleep(pollInterval - elapsedTime)
125 | fetchedRecords.asJava
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceTaskConfig.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc
2 |
3 | /**
4 | * @constructor
5 | * @param properties is set of configurations required to create JdbcSourceTaskConfig
6 | */
7 | class JdbcSourceTaskConfig(properties: Map[String, String]) extends JdbcSourceConnectorConfig(properties)
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/models/DatabaseProduct.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.models
2 |
3 | import enumeratum._
4 |
5 | import scala.collection.immutable.IndexedSeq
6 |
7 | /**
8 | * Database Product Name
9 | * ~~~~~~~~~~~~~~~~~~~~~
10 | *
11 | * MsSQL :: Microsoft SQL Server.
12 | *
13 | * MySQL :: MySQL Server.
14 | *
15 | */
16 | sealed abstract class DatabaseProduct(override val entryName: String) extends EnumEntry
17 |
18 | object DatabaseProduct extends Enum[DatabaseProduct] {
19 |
20 | val values: IndexedSeq[DatabaseProduct] = findValues
21 |
22 | case object MsSQL extends DatabaseProduct("Microsoft SQL Server")
23 | case object MySQL extends DatabaseProduct("MySQL")
24 | }
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/models/Mode.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.models
2 |
3 | import enumeratum._
4 |
5 | import scala.collection.immutable.IndexedSeq
6 |
7 | /**
8 | * Mode of operation
9 | * ~~~~~~~~~~~~~~~~~
10 | *
11 | * Timestamp Mode :: creation timestamp of a record is stored as offset.
12 | *
13 | * Incrementing Mode :: unique (auto) incrementing integral id of a record is stored as offset.
14 | *
15 | * Timestamp + Incrementing Mode :: pair of creation timestamp and unique (auto) incrementing integral id of a record
16 | * is stored as offset.
17 | */
18 | sealed abstract class Mode(override val entryName: String) extends EnumEntry
19 |
20 | object Mode extends Enum[Mode] {
21 |
22 | val values: IndexedSeq[Mode] = findValues
23 |
24 | case object TimestampMode extends Mode("timestamp")
25 | case object IncrementingMode extends Mode("incrementing")
26 | case object TimestampIncrementingMode extends Mode("timestamp+incrementing")
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/services/DataService.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.services
2 |
3 | import java.sql.{Connection, PreparedStatement, ResultSet}
4 |
5 | import com.agoda.kafka.connector.jdbc.utils.DataConverter
6 | import org.apache.kafka.connect.data.Schema
7 | import org.apache.kafka.connect.source.SourceRecord
8 |
9 | import scala.concurrent.duration.Duration
10 | import scala.util.Try
11 |
12 | trait DataService {
13 |
14 | /**
15 | * @return name of the stored procedure
16 | */
17 | def storedProcedureName: String
18 |
19 | /**
20 | * Utility to convert SQL ResultSet into Struct
21 | *
22 | * @return instance of DataConverter
23 | */
24 | def dataConverter: DataConverter
25 |
26 | /**
27 | * Fetch records from database
28 | *
29 | * @param connection database connection
30 | * @param timeout query timeout
31 | * @return Success(Seq(SourceRecord)) if records are processed successfully else Failure(Throwable)
32 | */
33 | def getRecords(connection: Connection, timeout: Duration): Try[Seq[SourceRecord]] = {
34 | for {
35 | preparedStatement <- createPreparedStatement(connection)
36 | resultSet <- executeStoredProcedure(preparedStatement, timeout)
37 | schema <- dataConverter.convertSchema(storedProcedureName, resultSet.getMetaData)
38 | records <- extractRecords(resultSet, schema)
39 | } yield records
40 | }
41 |
42 | protected def createPreparedStatement(connection: Connection): Try[PreparedStatement]
43 |
44 | protected def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]]
45 |
46 | private def executeStoredProcedure(preparedStatement: PreparedStatement, timeout: Duration): Try[ResultSet] = Try {
47 | preparedStatement.setQueryTimeout(timeout.toSeconds.toInt)
48 | preparedStatement.executeQuery
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/services/IdBasedDataService.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.services
2 |
3 | import java.io.IOException
4 | import java.sql.{Connection, PreparedStatement, ResultSet}
5 |
6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants
7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct
8 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL}
9 | import com.agoda.kafka.connector.jdbc.models.Mode.IncrementingMode
10 | import com.agoda.kafka.connector.jdbc.utils.DataConverter
11 | import org.apache.kafka.connect.data.Schema
12 | import org.apache.kafka.connect.data.Schema.Type
13 | import org.apache.kafka.connect.source.SourceRecord
14 | import org.slf4j.LoggerFactory
15 |
16 | import scala.collection.JavaConverters._
17 | import scala.collection.mutable.ListBuffer
18 | import scala.util.Try
19 |
20 | /**
21 | * @constructor
22 | * @param databaseProduct type of database server
23 | * @param storedProcedureName name of the stored procedure
24 | * @param batchSize number of records returned in each batch
25 | * @param batchSizeVariableName name of the batch size variable in stored procedure
26 | * @param incrementingVariableName name of the incrementing offset variable in stored procedure
27 | * @param incrementingOffset value of current incrementing offset
28 | * @param incrementingFieldName incrementing offset field name in returned records
29 | * @param topic name of kafka topic where records are stored
30 | * @param keyFieldOpt optional key field name in returned records
31 | * @param dataConverter ResultSet converter utility
32 | */
33 | case class IdBasedDataService(databaseProduct: DatabaseProduct,
34 | storedProcedureName: String,
35 | batchSize: Int,
36 | batchSizeVariableName: String,
37 | incrementingVariableName: String,
38 | var incrementingOffset: Long,
39 | incrementingFieldName: String,
40 | topic: String,
41 | keyFieldOpt: Option[String],
42 | dataConverter: DataConverter) extends DataService {
43 | private val logger = LoggerFactory.getLogger(this.getClass)
44 |
45 | override def createPreparedStatement(connection: Connection): Try[PreparedStatement] = Try {
46 | val preparedStatement = databaseProduct match {
47 | case MsSQL => connection.prepareStatement(s"EXECUTE $storedProcedureName @$incrementingVariableName = ?, @$batchSizeVariableName = ?")
48 | case MySQL => connection.prepareStatement(s"CALL $storedProcedureName (@$incrementingVariableName := ?, @$batchSizeVariableName := ?)")
49 | }
50 | preparedStatement.setObject(1, incrementingOffset)
51 | preparedStatement.setObject(2, batchSize)
52 | preparedStatement
53 | }
54 |
55 | override def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]] = Try {
56 | val sourceRecords = ListBuffer.empty[SourceRecord]
57 | val idSchemaType = schema.field(incrementingFieldName).schema.`type`()
58 | var max = incrementingOffset
59 | while(resultSet.next()) {
60 | dataConverter.convertRecord(schema, resultSet).map { record =>
61 | val id = idSchemaType match {
62 | case Type.INT8 => record.getInt8(incrementingFieldName).toLong
63 | case Type.INT16 => record.getInt16(incrementingFieldName).toLong
64 | case Type.INT32 => record.getInt32(incrementingFieldName).toLong
65 | case Type.INT64 => record.getInt64(incrementingFieldName).toLong
66 | case _ =>
67 | logger.warn("Id field is not of type INT")
68 | throw new IOException("Id field is not of type INT")
69 | }
70 | max = if (id > max) {
71 | keyFieldOpt match {
72 | case Some(keyField) =>
73 | sourceRecords += new SourceRecord(
74 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava,
75 | Map(IncrementingMode.entryName -> id).asJava, topic, null, schema, record.get(keyField), schema, record
76 | )
77 | case None =>
78 | sourceRecords += new SourceRecord(
79 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava,
80 | Map(IncrementingMode.entryName -> id).asJava, topic, schema, record
81 | )
82 | }
83 | id
84 | } else max
85 | }
86 | }
87 | incrementingOffset = max
88 | sourceRecords
89 | }
90 |
91 | override def toString: String = {
92 | s"""
93 | |{
94 | | "name" : "${this.getClass.getSimpleName}"
95 | | "mode" : "${IncrementingMode.entryName}"
96 | | "stored-procedure.name" : "$storedProcedureName"
97 | |}
98 | """.stripMargin
99 | }
100 | }
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/services/TimeBasedDataService.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.services
2 |
3 | import java.sql.{Connection, PreparedStatement, ResultSet, Timestamp}
4 | import java.util.{Date, GregorianCalendar, TimeZone}
5 |
6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants
7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct
8 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL}
9 | import com.agoda.kafka.connector.jdbc.models.Mode.TimestampMode
10 | import com.agoda.kafka.connector.jdbc.utils.DataConverter
11 | import org.apache.kafka.connect.data.Schema
12 | import org.apache.kafka.connect.source.SourceRecord
13 |
14 | import scala.collection.JavaConverters._
15 | import scala.collection.mutable.ListBuffer
16 | import scala.util.Try
17 |
18 | /**
19 | * @constructor
20 | * @param databaseProduct type of database server
21 | * @param storedProcedureName name of the stored procedure
22 | * @param batchSize number of records returned in each batch
23 | * @param batchSizeVariableName name of the batch size variable in stored procedure
24 | * @param timestampVariableName name of the timestamp offset variable in stored procedure
25 | * @param timestampOffset value of current timestamp offset
26 | * @param timestampFieldName timestamp offset field name in returned records
27 | * @param topic name of kafka topic where records are stored
28 | * @param keyFieldOpt optional key field name in returned records
29 | * @param dataConverter ResultSet converter utility
30 | */
31 | case class TimeBasedDataService(databaseProduct: DatabaseProduct,
32 | storedProcedureName: String,
33 | batchSize: Int,
34 | batchSizeVariableName: String,
35 | timestampVariableName: String,
36 | var timestampOffset: Long,
37 | timestampFieldName: String,
38 | topic: String,
39 | keyFieldOpt: Option[String],
40 | dataConverter: DataConverter,
41 | calendar: GregorianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"))
42 | ) extends DataService {
43 |
44 | override def createPreparedStatement(connection: Connection): Try[PreparedStatement] = Try {
45 | val preparedStatement = databaseProduct match {
46 | case MsSQL => connection.prepareStatement(s"EXECUTE $storedProcedureName @$timestampVariableName = ?, @$batchSizeVariableName = ?")
47 | case MySQL => connection.prepareStatement(s"CALL $storedProcedureName (@$timestampVariableName := ?, @$batchSizeVariableName := ?)")
48 | }
49 | preparedStatement.setTimestamp(1, new Timestamp(timestampOffset), calendar)
50 | preparedStatement.setObject(2, batchSize)
51 | preparedStatement
52 | }
53 |
54 | override def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]] = Try {
55 | val sourceRecords = ListBuffer.empty[SourceRecord]
56 | var max = timestampOffset
57 | while (resultSet.next()) {
58 | dataConverter.convertRecord(schema, resultSet) map { record =>
59 | val time = record.get(timestampFieldName).asInstanceOf[Date].getTime
60 | max = if(time > max) {
61 | keyFieldOpt match {
62 | case Some(keyField) =>
63 | sourceRecords += new SourceRecord(
64 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava,
65 | Map(TimestampMode.entryName -> time).asJava, topic, null, schema, record.get(keyField), schema, record
66 | )
67 | case None =>
68 | sourceRecords += new SourceRecord(
69 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava,
70 | Map(TimestampMode.entryName -> time).asJava, topic, schema, record
71 | )
72 | }
73 | time
74 | } else max
75 | }
76 | }
77 | timestampOffset = max
78 | sourceRecords
79 | }
80 |
81 | override def toString: String = {
82 | s"""
83 | |{
84 | | "name" : "${this.getClass.getSimpleName}"
85 | | "mode" : "${TimestampMode.entryName}"
86 | | "stored-procedure.name" : "$storedProcedureName"
87 | |}
88 | """.stripMargin
89 | }
90 | }
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/services/TimeIdBasedDataService.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.services
2 |
3 | import java.io.IOException
4 | import java.sql.{Connection, PreparedStatement, ResultSet, Timestamp}
5 | import java.util.{Date, GregorianCalendar, TimeZone}
6 |
7 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants
8 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct
9 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL}
10 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampMode}
11 | import com.agoda.kafka.connector.jdbc.utils.DataConverter
12 | import org.apache.kafka.connect.data.Schema
13 | import org.apache.kafka.connect.data.Schema.Type
14 | import org.apache.kafka.connect.source.SourceRecord
15 | import org.slf4j.LoggerFactory
16 |
17 | import scala.collection.JavaConverters._
18 | import scala.collection.mutable.ListBuffer
19 | import scala.util.Try
20 |
21 | /**
22 | * @constructor
23 | * @param databaseProduct type of database server
24 | * @param storedProcedureName name of the stored procedure
25 | * @param batchSize number of records returned in each batch
26 | * @param batchSizeVariableName name of the batch size variable in stored procedure
27 | * @param timestampVariableName name of the timestamp offset variable in stored procedure
28 | * @param timestampOffset value of current timestamp offset
29 | * @param timestampFieldName timestamp offset field name in returned records
30 | * @param incrementingVariableName name of the incrementing offset variable in stored procedure
31 | * @param incrementingOffset value of current incrementing offset
32 | * @param incrementingFieldName incrementing offset field name in returned records
33 | * @param topic name of kafka topic where records are stored
34 | * @param keyFieldOpt optional key field name in returned records
35 | * @param dataConverter ResultSet converter utility
36 | */
37 | case class TimeIdBasedDataService(databaseProduct: DatabaseProduct,
38 | storedProcedureName: String,
39 | batchSize: Int,
40 | batchSizeVariableName: String,
41 | timestampVariableName: String,
42 | var timestampOffset: Long,
43 | incrementingVariableName: String,
44 | var incrementingOffset: Long,
45 | timestampFieldName: String,
46 | incrementingFieldName: String,
47 | topic: String,
48 | keyFieldOpt: Option[String],
49 | dataConverter: DataConverter,
50 | calendar: GregorianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"))
51 | ) extends DataService {
52 | private val logger = LoggerFactory.getLogger(this.getClass)
53 |
54 | override def createPreparedStatement(connection: Connection): Try[PreparedStatement] = Try {
55 | val preparedStatement = databaseProduct match {
56 | case MsSQL => connection.prepareStatement(s"EXECUTE $storedProcedureName @$timestampVariableName = ?, @$incrementingVariableName = ?, @$batchSizeVariableName = ?")
57 | case MySQL => connection.prepareStatement(s"CALL $storedProcedureName (@$timestampVariableName := ?, @$incrementingVariableName := ?, @$batchSizeVariableName := ?)")
58 | }
59 | preparedStatement.setTimestamp(1, new Timestamp(timestampOffset), calendar)
60 | preparedStatement.setObject(2, incrementingOffset)
61 | preparedStatement.setObject(3, batchSize)
62 | preparedStatement
63 | }
64 |
65 | override def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]] = Try {
66 | val sourceRecords = ListBuffer.empty[SourceRecord]
67 | var maxTime = timestampOffset
68 | var maxId = incrementingOffset
69 | val idSchemaType = schema.field(incrementingFieldName).schema.`type`()
70 | while (resultSet.next()) {
71 | dataConverter.convertRecord(schema, resultSet) map { record =>
72 | val time = record.get(timestampFieldName).asInstanceOf[Date].getTime
73 | val id = idSchemaType match {
74 | case Type.INT8 => record.getInt8(incrementingFieldName).toLong
75 | case Type.INT16 => record.getInt16(incrementingFieldName).toLong
76 | case Type.INT32 => record.getInt32(incrementingFieldName).toLong
77 | case Type.INT64 => record.getInt64(incrementingFieldName).toLong
78 | case _ =>
79 | logger.warn("Id field is not of type INT")
80 | throw new IOException("Id field is not of type INT")
81 | }
82 |
83 | maxTime = if(time > maxTime) time else maxTime
84 | maxId = if (id > maxId) id else maxId
85 |
86 | if(time >= timestampOffset && id > incrementingOffset) {
87 | keyFieldOpt match {
88 | case Some(keyField) =>
89 | sourceRecords += new SourceRecord(
90 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava,
91 | Map(TimestampMode.entryName -> time, IncrementingMode.entryName -> id).asJava,
92 | topic, null, schema, record.get(keyField), schema, record
93 | )
94 | case None =>
95 | sourceRecords += new SourceRecord(
96 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava,
97 | Map(TimestampMode.entryName -> time, IncrementingMode.entryName -> id).asJava,
98 | topic, schema, record
99 | )
100 | }
101 | }
102 | }
103 | }
104 | timestampOffset = maxTime
105 | incrementingOffset = maxId
106 | sourceRecords
107 | }
108 |
109 | override def toString: String = {
110 | s"""
111 | |{
112 | | "name" : "${this.getClass.getSimpleName}"
113 | | "mode" : "${TimestampMode.entryName}+${IncrementingMode.entryName}"
114 | | "stored-procedure.name" : "$storedProcedureName"
115 | |}
116 | """.stripMargin
117 | }
118 | }
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/utils/DataConverter.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.utils
2 |
3 | import java.io.IOException
4 | import java.sql.{ResultSet, ResultSetMetaData, Types}
5 | import java.util.{GregorianCalendar, TimeZone}
6 |
7 | import com.agoda.kafka.connector.jdbc.JdbcSourceTask
8 | import org.apache.kafka.connect.data._
9 | import org.slf4j.LoggerFactory
10 |
11 | import scala.util.Try
12 |
13 | class DataConverter {
14 | private val logger = LoggerFactory.getLogger(this.getClass)
15 | private val UTC_CALENDAR = new GregorianCalendar(TimeZone.getTimeZone("UTC"))
16 |
17 | /**
18 | * Create schema from result set returned by stored procedure
19 | *
20 | * @param storedProcedureName name of the stored procedure, used as schema name
21 | * @param metadata metadata of result set returned by the stored procedure
22 | * @return Success(Schema) if schema is created successfully else Failure(Throwable)
23 | */
24 | def convertSchema(storedProcedureName: String, metadata: ResultSetMetaData): Try[Schema] = Try {
25 | val builder = SchemaBuilder.struct.name(storedProcedureName)
26 | (1 to metadata.getColumnCount).foreach(i => addFieldSchema(metadata, i, builder))
27 | builder.build
28 | }
29 |
30 | /**
31 | * Convert result set row into structured object
32 | *
33 | * @param schema schema created from result set metadata
34 | * @param resultSet result set returned by the stored procedure
35 | * @return Success(Struct) if structured object is created successfully else Failure(Throwable)
36 | */
37 | def convertRecord(schema: Schema, resultSet: ResultSet): Try[Struct] = Try {
38 | val metadata = resultSet.getMetaData
39 | val struct = new Struct(schema)
40 | (1 to metadata.getColumnCount).foreach { i =>
41 | convertFieldValue(resultSet, i, metadata.getColumnType(i), struct, metadata.getColumnLabel(i))
42 | }
43 | struct
44 | }
45 |
46 | private def addFieldSchema(metadata: ResultSetMetaData, col: Int, builder: SchemaBuilder) = {
47 | val label = metadata.getColumnLabel(col)
48 | val name = metadata.getColumnName(col)
49 | val fieldName = if (label != null && !label.isEmpty) label else name
50 | val sqlType = metadata.getColumnType(col)
51 | val optional = metadata.isNullable(col) == ResultSetMetaData.columnNullable ||
52 | metadata.isNullable(col) == ResultSetMetaData.columnNullableUnknown
53 |
54 | sqlType match {
55 | case Types.BOOLEAN =>
56 | if (optional) builder.field(fieldName, Schema.OPTIONAL_BOOLEAN_SCHEMA)
57 | else builder.field(fieldName, Schema.BOOLEAN_SCHEMA)
58 |
59 | case Types.BIT | Types.TINYINT =>
60 | if (optional) builder.field(fieldName, Schema.OPTIONAL_INT8_SCHEMA)
61 | else builder.field(fieldName, Schema.INT8_SCHEMA)
62 |
63 | case Types.SMALLINT =>
64 | if (optional) builder.field(fieldName, Schema.OPTIONAL_INT16_SCHEMA)
65 | else builder.field(fieldName, Schema.INT16_SCHEMA)
66 |
67 | case Types.INTEGER =>
68 | if (optional) builder.field(fieldName, Schema.OPTIONAL_INT32_SCHEMA)
69 | else builder.field(fieldName, Schema.INT32_SCHEMA)
70 |
71 | case Types.BIGINT =>
72 | if (optional) builder.field(fieldName, Schema.OPTIONAL_INT64_SCHEMA)
73 | else builder.field(fieldName, Schema.INT64_SCHEMA)
74 |
75 | case Types.REAL =>
76 | if (optional) builder.field(fieldName, Schema.OPTIONAL_FLOAT32_SCHEMA)
77 | else builder.field(fieldName, Schema.FLOAT32_SCHEMA)
78 |
79 | case Types.FLOAT | Types.DOUBLE =>
80 | if (optional) builder.field(fieldName, Schema.OPTIONAL_FLOAT64_SCHEMA)
81 | else builder.field(fieldName, Schema.FLOAT64_SCHEMA)
82 |
83 | case Types.NUMERIC | Types.DECIMAL =>
84 | val fieldBuilder = Decimal.builder(metadata.getScale(col))
85 | if (optional) fieldBuilder.optional
86 | builder.field(fieldName, fieldBuilder.build)
87 |
88 | case Types.CHAR | Types.VARCHAR | Types.LONGVARCHAR | Types.NCHAR | Types.NVARCHAR |
89 | Types.LONGNVARCHAR | Types.CLOB | Types.NCLOB | Types.DATALINK | Types.SQLXML =>
90 | if (optional) builder.field(fieldName, Schema.OPTIONAL_STRING_SCHEMA)
91 | else builder.field(fieldName, Schema.STRING_SCHEMA)
92 |
93 | case Types.BINARY | Types.BLOB | Types.VARBINARY | Types.LONGVARBINARY =>
94 | if (optional) builder.field(fieldName, Schema.OPTIONAL_BYTES_SCHEMA)
95 | else builder.field(fieldName, Schema.BYTES_SCHEMA)
96 |
97 | case Types.DATE =>
98 | val dateSchemaBuilder = Date.builder
99 | if (optional) dateSchemaBuilder.optional
100 | builder.field(fieldName, dateSchemaBuilder.build)
101 |
102 | case Types.TIME =>
103 | val timeSchemaBuilder = Time.builder
104 | if (optional) timeSchemaBuilder.optional
105 | builder.field(fieldName, timeSchemaBuilder.build)
106 |
107 | case Types.TIMESTAMP =>
108 | val tsSchemaBuilder = Timestamp.builder
109 | if (optional) tsSchemaBuilder.optional
110 | builder.field(fieldName, tsSchemaBuilder.build)
111 |
112 | case _ =>
113 | logger.warn("JDBC type {} not currently supported", sqlType)
114 | }
115 | }
116 |
117 | private def convertFieldValue(resultSet: ResultSet, col: Int, colType: Int, struct: Struct, fieldName: String) = {
118 | val colValue = colType match {
119 | case Types.BOOLEAN => resultSet.getBoolean(col)
120 |
121 | case Types.BIT | Types.TINYINT => resultSet.getByte(col)
122 |
123 | case Types.SMALLINT => resultSet.getShort(col)
124 |
125 | case Types.INTEGER => resultSet.getInt(col)
126 |
127 | case Types.BIGINT => resultSet.getLong(col)
128 |
129 | case Types.REAL => resultSet.getFloat(col)
130 |
131 | case Types.FLOAT | Types.DOUBLE => resultSet.getDouble(col)
132 |
133 | case Types.NUMERIC | Types.DECIMAL => resultSet.getBigDecimal(col)
134 |
135 | case Types.CHAR | Types.VARCHAR | Types.LONGVARCHAR => resultSet.getString(col)
136 |
137 | case Types.NCHAR | Types.NVARCHAR | Types.LONGNVARCHAR => resultSet.getNString(col)
138 |
139 | case Types.CLOB | Types.NCLOB =>
140 | val clob = if (colType == Types.CLOB) resultSet.getClob(col) else resultSet.getNClob(col)
141 | val bytes =
142 | if(clob == null) null
143 | else if(clob.length > Integer.MAX_VALUE) {
144 | logger.warn("Can't process CLOBs longer than Integer.MAX_VALUE")
145 | throw new IOException("Can't process CLOBs longer than Integer.MAX_VALUE")
146 | }
147 | else clob.getSubString(1, clob.length.toInt)
148 | if(clob != null) clob.free()
149 | bytes
150 |
151 | case Types.DATALINK =>
152 | val url = resultSet.getURL(col)
153 | if (url != null) url.toString else null
154 |
155 | case Types.SQLXML =>
156 | val xml = resultSet.getSQLXML(col)
157 | if (xml != null) xml.getString else null
158 |
159 | case Types.BINARY | Types.VARBINARY | Types.LONGVARBINARY => resultSet.getBytes(col)
160 |
161 | case Types.BLOB =>
162 | val blob = resultSet.getBlob(col)
163 | val bytes =
164 | if (blob == null) null
165 | else if (blob.length > Integer.MAX_VALUE) {
166 | logger.warn("Can't process BLOBs longer than Integer.MAX_VALUE")
167 | throw new IOException("Can't process BLOBs longer than Integer.MAX_VALUE")
168 | }
169 | else blob.getBytes(1, blob.length.toInt)
170 | if(blob != null) blob.free()
171 | bytes
172 |
173 | case Types.DATE => resultSet.getDate(col, UTC_CALENDAR)
174 |
175 | case Types.TIME => resultSet.getTime(col, UTC_CALENDAR)
176 |
177 | case Types.TIMESTAMP => resultSet.getTimestamp(col, UTC_CALENDAR)
178 |
179 | case _ => null
180 | }
181 |
182 | struct.put(fieldName, if (resultSet.wasNull) null else colValue)
183 | }
184 | }
--------------------------------------------------------------------------------
/src/main/scala/com/agoda/kafka/connector/jdbc/utils/Version.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.utils
2 |
3 | import com.agoda.BuildInfo
4 |
5 | object Version {
6 | lazy val getVersion: String = BuildInfo.version
7 | }
8 |
--------------------------------------------------------------------------------
/src/test/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnectorConfigTest.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc
2 |
3 | import com.agoda.kafka.connector.jdbc.models.Mode.TimestampIncrementingMode
4 | import org.scalatest.{Matchers, WordSpec}
5 |
6 | class JdbcSourceConnectorConfigTest extends WordSpec with Matchers {
7 | import JdbcSourceConnectorConfigTestData._
8 |
9 | "JDBC Source Connector Config" should {
10 |
11 | "throw exception if any mandatory configuration is missing" in {
12 | val properties = Map(connectionUrlProperty, incrementingModeProperty, topicProperty,
13 | pollIntervalProperty, batchVariableNameProperty, incrementingVariableNameConfig, incrementingFieldNameConfig)
14 |
15 | the [IllegalArgumentException] thrownBy new JdbcSourceConnectorConfig(properties).getClass
16 | }
17 |
18 | "create JDBC source configuration for incrementing mode" in {
19 | val properties = Map(connectionUrlProperty, incrementingModeProperty, storedProcedureProperty, topicProperty,
20 | pollIntervalProperty, batchVariableNameProperty, incrementingVariableNameConfig, incrementingFieldNameConfig)
21 |
22 | new JdbcSourceConnectorConfig(properties).getClass shouldEqual classOf[JdbcSourceConnectorConfig]
23 | }
24 |
25 | "throw exception if any configuration for incrementing mode is missing" in {
26 | val properties = Map(connectionUrlProperty, incrementingModeProperty, storedProcedureProperty, topicProperty,
27 | pollIntervalProperty, batchVariableNameProperty, incrementingVariableNameConfig)
28 |
29 | the [IllegalArgumentException] thrownBy new JdbcSourceConnectorConfig(properties).getClass
30 | }
31 |
32 | "create JDBC source configuration for timestamp mode" in {
33 | val properties = Map(connectionUrlProperty, timestampModeProperty, storedProcedureProperty, topicProperty,
34 | pollIntervalProperty, batchVariableNameProperty, timestampVariableNameConfig, timestampFieldNameConfig)
35 |
36 | new JdbcSourceConnectorConfig(properties).getClass shouldEqual classOf[JdbcSourceConnectorConfig]
37 | }
38 |
39 | "throw exception if any configuration for timestamp mode is missing" in {
40 | val properties = Map(connectionUrlProperty, timestampModeProperty, storedProcedureProperty, topicProperty,
41 | pollIntervalProperty, batchVariableNameProperty, timestampFieldNameConfig)
42 |
43 | the [IllegalArgumentException] thrownBy new JdbcSourceConnectorConfig(properties).getClass
44 | }
45 |
46 | "create JDBC source configuration for timestamp+incrementing mode" in {
47 | val properties = Map(connectionUrlProperty, timestampIncrementingModeProperty, storedProcedureProperty,
48 | topicProperty, pollIntervalProperty, batchVariableNameProperty, timestampVariableNameConfig, timestampFieldNameConfig,
49 | incrementingVariableNameConfig, incrementingFieldNameConfig)
50 |
51 | new JdbcSourceConnectorConfig(properties).getClass shouldEqual classOf[JdbcSourceConnectorConfig]
52 | }
53 |
54 | "throw exception if any configuration for timestamp+incrementing mode is missing" in {
55 | val properties = Map(connectionUrlProperty, timestampIncrementingModeProperty, storedProcedureProperty,
56 | topicProperty, pollIntervalProperty, batchVariableNameProperty, timestampFieldNameConfig,
57 | incrementingVariableNameConfig)
58 |
59 | the [IllegalArgumentException] thrownBy new JdbcSourceConnectorConfig(properties).getClass
60 | }
61 |
62 | "get all properties from configuration" in {
63 | val properties = Map(connectionUrlProperty, timestampIncrementingModeProperty, storedProcedureProperty,
64 | topicProperty, pollIntervalProperty, batchSizeProperty, batchVariableNameProperty, timestampVariableNameConfig,
65 | timestampFieldNameConfig, timestampOffsetConfig, incrementingVariableNameConfig, incrementingFieldNameConfig,
66 | incrementingOffsetConfig, keyFieldConfig)
67 |
68 | val configuration = new JdbcSourceConnectorConfig(properties)
69 |
70 | configuration.getConnectionUrl shouldBe "test-connection"
71 | configuration.getMode shouldBe TimestampIncrementingMode
72 | configuration.getStoredProcedureName shouldBe "test-procedure"
73 | configuration.getTopic shouldBe "test-topic"
74 | configuration.getPollInterval shouldBe 100
75 | configuration.getMaxBatchSize shouldBe 1000
76 | configuration.getMaxBatchSizeVariableName shouldBe "batch"
77 | configuration.getTimestampVariableName shouldBe Some("time")
78 | configuration.getTimestampFieldName shouldBe Some("time")
79 | configuration.getIncrementingVariableName shouldBe Some("id")
80 | configuration.getIncrementingFieldName shouldBe Some("id")
81 | configuration.getTimestampOffset shouldBe 946684800000L
82 | configuration.getIncrementingOffset shouldBe 5L
83 | configuration.getKeyField shouldBe Some("test-key")
84 | }
85 |
86 | "get optional properties with default values from configuration" in {
87 | val properties = Map(connectionUrlProperty, timestampIncrementingModeProperty, storedProcedureProperty,
88 | topicProperty, batchVariableNameProperty, timestampVariableNameConfig, timestampFieldNameConfig,
89 | incrementingVariableNameConfig, incrementingFieldNameConfig)
90 |
91 | val configuration = new JdbcSourceConnectorConfig(properties)
92 |
93 | configuration.getPollInterval shouldBe 5000
94 | configuration.getMaxBatchSize shouldBe 100
95 | configuration.getTimestampOffset shouldBe 0L
96 | configuration.getIncrementingOffset shouldBe 0L
97 | }
98 | }
99 | }
100 |
101 | object JdbcSourceConnectorConfigTestData {
102 | val connectionUrlProperty: (String, String) = "connection.url" -> "test-connection"
103 | val timestampModeProperty: (String, String) = "mode" -> "timestamp"
104 | val incrementingModeProperty: (String, String) = "mode" -> "incrementing"
105 | val timestampIncrementingModeProperty: (String, String) = "mode" -> "timestamp+incrementing"
106 | val storedProcedureProperty: (String, String) = "stored-procedure.name" -> "test-procedure"
107 | val topicProperty: (String, String) = "topic" -> "test-topic"
108 | val pollIntervalProperty: (String, String) = "poll.interval.ms" -> "100"
109 | val batchSizeProperty: (String, String) = "batch.max.records" -> "1000"
110 | val batchVariableNameProperty: (String, String) = "batch.max.rows.variable.name" -> "batch"
111 | val timestampVariableNameConfig: (String, String) = "timestamp.variable.name" -> "time"
112 | val timestampFieldNameConfig: (String, String) = "timestamp.field.name" -> "time"
113 | val timestampOffsetConfig: (String, String) = "timestamp.offset" -> "2000-01-01 00:00:00"
114 | val incrementingVariableNameConfig: (String, String) = "incrementing.variable.name" -> "id"
115 | val incrementingFieldNameConfig: (String, String) = "incrementing.field.name" -> "id"
116 | val incrementingOffsetConfig: (String, String) = "incrementing.offset" -> "5"
117 | val keyFieldConfig: (String, String) = "key.field.name" -> "test-key"
118 | }
119 |
--------------------------------------------------------------------------------
/src/test/scala/com/agoda/kafka/connector/jdbc/models/DatabaseProductTest.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.models
2 |
3 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL}
4 | import org.scalatest.{Matchers, WordSpec}
5 |
6 | class DatabaseProductTest extends WordSpec with Matchers {
7 |
8 | "module" should {
9 | "convert DatabaseProduct to its string representation" in {
10 | DatabaseProduct.MySQL.entryName shouldEqual "MySQL"
11 | DatabaseProduct.MsSQL.entryName shouldEqual "Microsoft SQL Server"
12 | }
13 |
14 | "convert string to corresponding DatabaseProduct representation" in {
15 | DatabaseProduct.withName("MySQL") shouldBe MySQL
16 | DatabaseProduct.withName("Microsoft SQL Server") shouldBe MsSQL
17 | }
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/test/scala/com/agoda/kafka/connector/jdbc/models/ModeTest.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.models
2 |
3 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampIncrementingMode, TimestampMode}
4 | import org.scalatest.{Matchers, WordSpec}
5 |
6 | class ModeTest extends WordSpec with Matchers {
7 |
8 | "module" should {
9 | "convert Mode to its string representation" in {
10 | Mode.TimestampMode.entryName shouldEqual "timestamp"
11 | Mode.IncrementingMode.entryName shouldEqual "incrementing"
12 | Mode.TimestampIncrementingMode.entryName shouldEqual "timestamp+incrementing"
13 | }
14 |
15 | "convert string to corresponding Mode representation" in {
16 | Mode.withName("timestamp") shouldBe TimestampMode
17 | Mode.withName("incrementing") shouldBe IncrementingMode
18 | Mode.withName("timestamp+incrementing") shouldBe TimestampIncrementingMode
19 | }
20 | }
21 | }
22 |
--------------------------------------------------------------------------------
/src/test/scala/com/agoda/kafka/connector/jdbc/services/DataServiceTest.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.services
2 |
3 | import java.sql.{Connection, PreparedStatement, ResultSet, ResultSetMetaData}
4 |
5 | import com.agoda.kafka.connector.jdbc.utils.DataConverter
6 | import org.apache.kafka.connect.data.Schema
7 | import org.apache.kafka.connect.source.SourceRecord
8 | import org.scalatest.mockito.MockitoSugar
9 | import org.mockito.Mockito._
10 | import org.scalatest.{Matchers, WordSpec}
11 |
12 | import scala.concurrent.duration._
13 | import scala.util.Success
14 |
15 | class DataServiceTest extends WordSpec with Matchers with MockitoSugar {
16 |
17 | "Data Service" should {
18 |
19 | val spName = "stored-procedure"
20 | val connection = mock[Connection]
21 | val converter = mock[DataConverter]
22 | val sourceRecord1 = mock[SourceRecord]
23 | val sourceRecord2 = mock[SourceRecord]
24 | val resultSet = mock[ResultSet]
25 | val resultSetMetadata = mock[ResultSetMetaData]
26 | val preparedStatement = mock[PreparedStatement]
27 | val schema = mock[Schema]
28 |
29 | val dataService = new DataService {
30 |
31 | override def storedProcedureName: String = spName
32 |
33 | override protected def createPreparedStatement(connection: Connection) = Success(preparedStatement)
34 |
35 | override protected def extractRecords(resultSet: ResultSet, schema: Schema) = Success(Seq(sourceRecord1, sourceRecord2))
36 |
37 | override def dataConverter: DataConverter = converter
38 | }
39 |
40 | "get records" in {
41 | doNothing().when(preparedStatement).setQueryTimeout(1)
42 | when(preparedStatement.executeQuery).thenReturn(resultSet)
43 | when(resultSet.getMetaData).thenReturn(resultSetMetadata)
44 | when(converter.convertSchema(spName, resultSetMetadata)).thenReturn(Success(schema))
45 |
46 | dataService.getRecords(connection, 1.second) shouldBe Success(Seq(sourceRecord1, sourceRecord2))
47 |
48 | verify(preparedStatement).setQueryTimeout(1)
49 | verify(preparedStatement).executeQuery
50 | verify(resultSet).getMetaData
51 | verify(converter).convertSchema(spName, resultSetMetadata)
52 | }
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/test/scala/com/agoda/kafka/connector/jdbc/services/IdBasedDataServiceTest.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.services
2 |
3 | import java.sql.{Connection, PreparedStatement, ResultSet}
4 |
5 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants
6 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL}
7 | import com.agoda.kafka.connector.jdbc.models.Mode.IncrementingMode
8 | import com.agoda.kafka.connector.jdbc.utils.DataConverter
9 | import org.apache.kafka.connect.data.{Field, Schema, Struct}
10 | import org.apache.kafka.connect.source.SourceRecord
11 | import org.mockito.Mockito._
12 | import org.scalatest.mockito.MockitoSugar
13 | import org.scalatest.{Matchers, WordSpec}
14 |
15 | import scala.collection.JavaConverters._
16 | import scala.collection.mutable.ListBuffer
17 | import scala.util.Success
18 |
19 | class IdBasedDataServiceTest extends WordSpec with Matchers with MockitoSugar {
20 |
21 | "ID Based Data Service" should {
22 |
23 | val dataConverter = mock[DataConverter]
24 |
25 | val idBasedDataServiceMssql =
26 | IdBasedDataService(
27 | databaseProduct = MsSQL,
28 | storedProcedureName = "stored-procedure",
29 | batchSize = 100,
30 | batchSizeVariableName = "batch-size-variable",
31 | incrementingVariableName = "incrementing-variable",
32 | incrementingOffset = 0L,
33 | incrementingFieldName = "id",
34 | topic = "id-based-data-topic",
35 | keyFieldOpt = None,
36 | dataConverter = dataConverter
37 | )
38 |
39 | val idBasedDataServiceMysql =
40 | IdBasedDataService(
41 | databaseProduct = MySQL,
42 | storedProcedureName = "stored-procedure",
43 | batchSize = 100,
44 | batchSizeVariableName = "batch-size-variable",
45 | incrementingVariableName = "incrementing-variable",
46 | incrementingOffset = 0L,
47 | incrementingFieldName = "id",
48 | topic = "id-based-data-topic",
49 | keyFieldOpt = None,
50 | dataConverter = dataConverter
51 | )
52 |
53 | "create correct prepared statement for Mssql" in {
54 | val connection = mock[Connection]
55 | val statement = mock[PreparedStatement]
56 |
57 | when(connection.prepareStatement("EXECUTE stored-procedure @incrementing-variable = ?, @batch-size-variable = ?")).thenReturn(statement)
58 | doNothing().when(statement).setObject(1, 0L)
59 | doNothing().when(statement).setObject(2, 100)
60 |
61 | idBasedDataServiceMssql.createPreparedStatement(connection)
62 |
63 | verify(connection).prepareStatement("EXECUTE stored-procedure @incrementing-variable = ?, @batch-size-variable = ?")
64 | verify(statement).setObject(1, 0L)
65 | verify(statement).setObject(2, 100)
66 | }
67 |
68 | "create correct prepared statement for Mysql" in {
69 | val connection = mock[Connection]
70 | val statement = mock[PreparedStatement]
71 |
72 | when(connection.prepareStatement("CALL stored-procedure (@incrementing-variable := ?, @batch-size-variable := ?)")).thenReturn(statement)
73 | doNothing().when(statement).setObject(1, 0L)
74 | doNothing().when(statement).setObject(2, 100)
75 |
76 | idBasedDataServiceMysql.createPreparedStatement(connection)
77 |
78 | verify(connection).prepareStatement("CALL stored-procedure (@incrementing-variable := ?, @batch-size-variable := ?)")
79 | verify(statement).setObject(1, 0L)
80 | verify(statement).setObject(2, 100)
81 | }
82 |
83 | "create correct string representation" in {
84 | idBasedDataServiceMssql.toString shouldBe
85 | s"""
86 | |{
87 | | "name" : "IdBasedDataService"
88 | | "mode" : "incrementing"
89 | | "stored-procedure.name" : "stored-procedure"
90 | |}
91 | """.stripMargin
92 | }
93 |
94 | "extract records" in {
95 | val resultSet = mock[ResultSet]
96 | val schema = mock[Schema]
97 | val field = mock[Field]
98 | val struct = mock[Struct]
99 |
100 | when(schema.field("id")).thenReturn(field)
101 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA)
102 | when(resultSet.next()).thenReturn(true, true, false)
103 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
104 | when(struct.getInt32("id")).thenReturn(1, 2)
105 |
106 |
107 | idBasedDataServiceMssql.extractRecords(resultSet, schema).toString shouldBe
108 | Success(
109 | ListBuffer(
110 | new SourceRecord(
111 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
112 | Map(IncrementingMode.entryName -> 1).asJava, "id-based-data-topic", schema, struct
113 | ),
114 | new SourceRecord(
115 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
116 | Map(IncrementingMode.entryName -> 2).asJava, "id-based-data-topic", schema, struct
117 | )
118 | )
119 | ).toString
120 | idBasedDataServiceMssql.incrementingOffset shouldBe 2
121 | }
122 |
123 | "return nothing when record ids are smaller than or equal to the offset id" in {
124 | val idBasedDataServiceWithLargerOffsetMssql = idBasedDataServiceMssql.copy(incrementingOffset = 3L)
125 | val resultSet = mock[ResultSet]
126 | val schema = mock[Schema]
127 | val field = mock[Field]
128 | val struct = mock[Struct]
129 |
130 | when(schema.field("id")).thenReturn(field)
131 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA)
132 | when(resultSet.next()).thenReturn(true, true, true, false)
133 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
134 | when(struct.getInt32("id")).thenReturn(1, 2, 3)
135 |
136 | idBasedDataServiceWithLargerOffsetMssql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer())
137 | idBasedDataServiceWithLargerOffsetMssql.incrementingOffset shouldBe 3L
138 | }
139 |
140 | "extract records with key" in {
141 | val idBasedDataServiceWithKeyMysql = idBasedDataServiceMysql.copy(keyFieldOpt = Some("key"))
142 | val resultSet = mock[ResultSet]
143 | val schema = mock[Schema]
144 | val field = mock[Field]
145 | val struct = mock[Struct]
146 |
147 | when(schema.field("id")).thenReturn(field)
148 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA)
149 | when(resultSet.next()).thenReturn(true, true, false)
150 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
151 | when(struct.getInt32("id")).thenReturn(1, 2)
152 | when(struct.get("key")).thenReturn("key-1", "key-2")
153 |
154 |
155 | idBasedDataServiceWithKeyMysql.extractRecords(resultSet, schema).toString shouldBe
156 | Success(
157 | ListBuffer(
158 | new SourceRecord(
159 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
160 | Map(IncrementingMode.entryName -> 1).asJava, "id-based-data-topic", null, schema, "key-1", schema, struct
161 | ),
162 | new SourceRecord(
163 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
164 | Map(IncrementingMode.entryName -> 2).asJava, "id-based-data-topic", null, schema, "key-2", schema, struct
165 | )
166 | )
167 | ).toString
168 | }
169 |
170 | "returns no record when id field is not of type integer" in {
171 | val resultSet = mock[ResultSet]
172 | val schema = mock[Schema]
173 | val field = mock[Field]
174 | val struct = mock[Struct]
175 |
176 | when(schema.field("id")).thenReturn(field)
177 | when(field.schema()).thenReturn(Schema.STRING_SCHEMA)
178 | when(resultSet.next()).thenReturn(true, false)
179 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
180 |
181 | idBasedDataServiceMysql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer())
182 | }
183 | }
184 | }
185 |
--------------------------------------------------------------------------------
/src/test/scala/com/agoda/kafka/connector/jdbc/services/TimeBasedDataServiceTest.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.services
2 |
3 | import java.sql._
4 | import java.util.{GregorianCalendar, TimeZone}
5 |
6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants
7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL}
8 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampMode}
9 | import com.agoda.kafka.connector.jdbc.utils.DataConverter
10 | import org.apache.kafka.connect.data.{Field, Schema, Struct}
11 | import org.apache.kafka.connect.source.SourceRecord
12 | import org.mockito.Mockito._
13 | import org.scalatest.mockito.MockitoSugar
14 | import org.scalatest.{Matchers, WordSpec}
15 |
16 | import scala.collection.JavaConverters._
17 | import scala.collection.mutable.ListBuffer
18 | import scala.util.Success
19 |
20 | class TimeBasedDataServiceTest extends WordSpec with Matchers with MockitoSugar {
21 |
22 | "Time Based Data Service" should {
23 |
24 | val dataConverter = mock[DataConverter]
25 | val UTC_Calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC"))
26 |
27 | val timeBasedDataServiceMssql =
28 | TimeBasedDataService(
29 | databaseProduct = MsSQL,
30 | storedProcedureName = "stored-procedure",
31 | batchSize = 100,
32 | batchSizeVariableName = "batch-size-variable",
33 | timestampVariableName = "timestamp-variable",
34 | timestampOffset = 0L,
35 | timestampFieldName = "time",
36 | topic = "time-based-data-topic",
37 | keyFieldOpt = None,
38 | dataConverter = dataConverter,
39 | calendar = UTC_Calendar
40 | )
41 |
42 | val timeBasedDataServiceMysql =
43 | TimeBasedDataService(
44 | databaseProduct = MySQL,
45 | storedProcedureName = "stored-procedure",
46 | batchSize = 100,
47 | batchSizeVariableName = "batch-size-variable",
48 | timestampVariableName = "timestamp-variable",
49 | timestampOffset = 0L,
50 | timestampFieldName = "time",
51 | topic = "time-based-data-topic",
52 | keyFieldOpt = None,
53 | dataConverter = dataConverter,
54 | calendar = UTC_Calendar
55 | )
56 |
57 | val timestamp = new Timestamp(0L)
58 |
59 | "create correct prepared statement for Mssql" in {
60 | val connection = mock[Connection]
61 | val statement = mock[PreparedStatement]
62 |
63 | when(connection.prepareStatement("EXECUTE stored-procedure @timestamp-variable = ?, @batch-size-variable = ?")).thenReturn(statement)
64 | doNothing().when(statement).setTimestamp(1, timestamp, UTC_Calendar)
65 | doNothing().when(statement).setObject(2, 100)
66 |
67 | timeBasedDataServiceMssql.createPreparedStatement(connection)
68 |
69 | verify(connection).prepareStatement("EXECUTE stored-procedure @timestamp-variable = ?, @batch-size-variable = ?")
70 | verify(statement).setTimestamp(1, timestamp, UTC_Calendar)
71 | verify(statement).setObject(2, 100)
72 | }
73 |
74 | "create correct prepared statement for Mysql" in {
75 | val connection = mock[Connection]
76 | val statement = mock[PreparedStatement]
77 |
78 | when(connection.prepareStatement("CALL stored-procedure (@timestamp-variable := ?, @batch-size-variable := ?)")).thenReturn(statement)
79 | doNothing().when(statement).setTimestamp(1, timestamp, UTC_Calendar)
80 | doNothing().when(statement).setObject(2, 100)
81 |
82 | timeBasedDataServiceMysql.createPreparedStatement(connection)
83 |
84 | verify(connection).prepareStatement("CALL stored-procedure (@timestamp-variable := ?, @batch-size-variable := ?)")
85 | verify(statement).setTimestamp(1, timestamp, UTC_Calendar)
86 | verify(statement).setObject(2, 100)
87 | }
88 |
89 | "create correct string representation" in {
90 | timeBasedDataServiceMssql.toString shouldBe
91 | s"""
92 | |{
93 | | "name" : "TimeBasedDataService"
94 | | "mode" : "timestamp"
95 | | "stored-procedure.name" : "stored-procedure"
96 | |}
97 | """.stripMargin
98 | }
99 |
100 | "extract records" in {
101 | val resultSet = mock[ResultSet]
102 | val schema = mock[Schema]
103 | val struct = mock[Struct]
104 |
105 | when(resultSet.next()).thenReturn(true, true, false)
106 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
107 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L))
108 |
109 | timeBasedDataServiceMssql.extractRecords(resultSet, schema).toString shouldBe
110 | Success(
111 | ListBuffer(
112 | new SourceRecord(
113 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
114 | Map(TimestampMode.entryName -> 1L).asJava, "time-based-data-topic", schema, struct
115 | ),
116 | new SourceRecord(
117 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
118 | Map(TimestampMode.entryName -> 2L).asJava, "time-based-data-topic", schema, struct
119 | )
120 | )
121 | ).toString
122 | timeBasedDataServiceMssql.timestampOffset shouldBe 2L
123 | }
124 |
125 | "return nothing when the record timestamps are smaller than or equal to the offset timestamp" in {
126 | val timeBasedDataServiceWithLargerOffsetMssql = timeBasedDataServiceMssql.copy(timestampOffset = 3L)
127 | val resultSet = mock[ResultSet]
128 | val schema = mock[Schema]
129 | val struct = mock[Struct]
130 |
131 | when(resultSet.next()).thenReturn(true, true, true, false)
132 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
133 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L), new Date(3L))
134 |
135 | timeBasedDataServiceWithLargerOffsetMssql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer())
136 | timeBasedDataServiceWithLargerOffsetMssql.timestampOffset shouldBe 3L
137 | }
138 |
139 | "extract records with key" in {
140 | val timeBasedDataServiceWithKeyMysql = timeBasedDataServiceMysql.copy(keyFieldOpt = Some("key"))
141 | val resultSet = mock[ResultSet]
142 | val schema = mock[Schema]
143 | val struct = mock[Struct]
144 |
145 | when(resultSet.next()).thenReturn(true, true, false)
146 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
147 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L))
148 | when(struct.get("key")).thenReturn("key-1", "key-2")
149 |
150 | timeBasedDataServiceWithKeyMysql.extractRecords(resultSet, schema).toString shouldBe
151 | Success(
152 | ListBuffer(
153 | new SourceRecord(
154 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
155 | Map(TimestampMode.entryName -> 1L).asJava, "time-based-data-topic", null, schema, "key-1", schema, struct
156 | ),
157 | new SourceRecord(
158 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
159 | Map(TimestampMode.entryName -> 2L).asJava, "time-based-data-topic", null, schema, "key-2", schema, struct
160 | )
161 | )
162 | ).toString
163 | }
164 | }
165 | }
166 |
167 |
--------------------------------------------------------------------------------
/src/test/scala/com/agoda/kafka/connector/jdbc/services/TimeIdBasedDataServiceTest.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.services
2 |
3 | import java.sql._
4 | import java.util.{GregorianCalendar, TimeZone}
5 |
6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants
7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL}
8 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampMode}
9 | import com.agoda.kafka.connector.jdbc.utils.DataConverter
10 | import org.apache.kafka.connect.data.{Field, Schema, Struct}
11 | import org.apache.kafka.connect.source.SourceRecord
12 | import org.mockito.Mockito._
13 | import org.scalatest.mockito.MockitoSugar
14 | import org.scalatest.{Matchers, WordSpec}
15 |
16 | import scala.collection.JavaConverters._
17 | import scala.collection.mutable.ListBuffer
18 | import scala.util.Success
19 |
20 | class TimeIdBasedDataServiceTest extends WordSpec with Matchers with MockitoSugar {
21 |
22 | "Time ID Based Data Service" should {
23 |
24 | val dataConverter = mock[DataConverter]
25 |
26 | val UTC_CALENDAR = new GregorianCalendar(TimeZone.getTimeZone("UTC"))
27 |
28 | val timeIdBasedDataServiceMssql =
29 | TimeIdBasedDataService(
30 | databaseProduct = MsSQL,
31 | storedProcedureName = "stored-procedure",
32 | batchSize = 100,
33 | batchSizeVariableName = "batch-size-variable",
34 | timestampVariableName = "timestamp-variable",
35 | timestampOffset = 0L,
36 | timestampFieldName = "time",
37 | incrementingVariableName = "incrementing-variable",
38 | incrementingOffset = 0L,
39 | incrementingFieldName = "id",
40 | topic = "time-id-based-data-topic",
41 | keyFieldOpt = None,
42 | dataConverter = dataConverter,
43 | calendar = UTC_CALENDAR
44 | )
45 |
46 | val timeIdBasedDataServiceMysql =
47 | TimeIdBasedDataService(
48 | databaseProduct = MySQL,
49 | storedProcedureName = "stored-procedure",
50 | batchSize = 100,
51 | batchSizeVariableName = "batch-size-variable",
52 | timestampVariableName = "timestamp-variable",
53 | timestampOffset = 0L,
54 | timestampFieldName = "time",
55 | incrementingVariableName = "incrementing-variable",
56 | incrementingOffset = 0L,
57 | incrementingFieldName = "id",
58 | topic = "time-id-based-data-topic",
59 | keyFieldOpt = None,
60 | dataConverter = dataConverter,
61 | calendar = UTC_CALENDAR
62 | )
63 |
64 | val timestamp = new Timestamp(0L)
65 |
66 | "create correct prepared statement for Mssql" in {
67 | val connection = mock[Connection]
68 | val statement = mock[PreparedStatement]
69 |
70 | when(connection.prepareStatement("EXECUTE stored-procedure @timestamp-variable = ?, @incrementing-variable = ?, @batch-size-variable = ?")).thenReturn(statement)
71 | doNothing().when(statement).setTimestamp(1, timestamp, UTC_CALENDAR)
72 | doNothing().when(statement).setObject(2, 0L)
73 | doNothing().when(statement).setObject(3, 100)
74 |
75 | timeIdBasedDataServiceMssql.createPreparedStatement(connection)
76 |
77 | verify(connection).prepareStatement("EXECUTE stored-procedure @timestamp-variable = ?, @incrementing-variable = ?, @batch-size-variable = ?")
78 | verify(statement).setTimestamp(1, timestamp, UTC_CALENDAR)
79 | verify(statement).setObject(2, 0L)
80 | verify(statement).setObject(3, 100)
81 | }
82 |
83 | "create correct prepared statement for Mysql" in {
84 | val connection = mock[Connection]
85 | val statement = mock[PreparedStatement]
86 |
87 | when(connection.prepareStatement("CALL stored-procedure (@timestamp-variable := ?, @incrementing-variable := ?, @batch-size-variable := ?)")).thenReturn(statement)
88 | doNothing().when(statement).setTimestamp(1, timestamp, UTC_CALENDAR)
89 | doNothing().when(statement).setObject(2, 0L)
90 | doNothing().when(statement).setObject(3, 100)
91 |
92 | timeIdBasedDataServiceMysql.createPreparedStatement(connection)
93 |
94 | verify(connection).prepareStatement("CALL stored-procedure (@timestamp-variable := ?, @incrementing-variable := ?, @batch-size-variable := ?)")
95 | verify(statement).setTimestamp(1, timestamp, UTC_CALENDAR)
96 | verify(statement).setObject(2, 0L)
97 | verify(statement).setObject(3, 100)
98 | }
99 |
100 | "create correct string representation" in {
101 | timeIdBasedDataServiceMssql.toString shouldBe
102 | s"""
103 | |{
104 | | "name" : "TimeIdBasedDataService"
105 | | "mode" : "timestamp+incrementing"
106 | | "stored-procedure.name" : "stored-procedure"
107 | |}
108 | """.stripMargin
109 | }
110 |
111 | "extract records" in {
112 | val resultSet = mock[ResultSet]
113 | val schema = mock[Schema]
114 | val field = mock[Field]
115 | val struct = mock[Struct]
116 |
117 | when(schema.field("id")).thenReturn(field)
118 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA)
119 | when(resultSet.next()).thenReturn(true, true, false)
120 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
121 | when(struct.getInt32("id")).thenReturn(1, 2)
122 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L))
123 |
124 | timeIdBasedDataServiceMssql.extractRecords(resultSet, schema).toString shouldBe
125 | Success(
126 | ListBuffer(
127 | new SourceRecord(
128 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
129 | Map(TimestampMode.entryName -> 1L, IncrementingMode.entryName -> 1).asJava,
130 | "time-id-based-data-topic", schema, struct
131 | ),
132 | new SourceRecord(
133 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
134 | Map(TimestampMode.entryName -> 2L, IncrementingMode.entryName -> 2).asJava,
135 | "time-id-based-data-topic", schema, struct
136 | )
137 | )
138 | ).toString
139 | timeIdBasedDataServiceMssql.timestampOffset shouldBe 2L
140 | timeIdBasedDataServiceMssql.incrementingOffset shouldBe 2
141 | }
142 |
143 | "return nothing when record ids and timestamps are smaller than offset id" in {
144 | val timeIdBasedDataServiceWithLargerOffsetMssql = timeIdBasedDataServiceMssql.copy(incrementingOffset = 3, timestampOffset = 3L)
145 | val resultSet = mock[ResultSet]
146 | val schema = mock[Schema]
147 | val field = mock[Field]
148 | val struct = mock[Struct]
149 |
150 | when(schema.field("id")).thenReturn(field)
151 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA)
152 | when(resultSet.next()).thenReturn(true, true, false)
153 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
154 | when(struct.getInt32("id")).thenReturn(1, 2)
155 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L))
156 |
157 | timeIdBasedDataServiceWithLargerOffsetMssql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer())
158 |
159 | timeIdBasedDataServiceWithLargerOffsetMssql.timestampOffset shouldBe 3L
160 | timeIdBasedDataServiceWithLargerOffsetMssql.incrementingOffset shouldBe 3
161 | }
162 |
163 | "return record when the record timestamps are equals but record ids are than offset id" in {
164 | val timeIdBasedDataServiceWithLargerOffsetMssql = timeIdBasedDataServiceMssql.copy(incrementingOffset = 3, timestampOffset = 3L)
165 | val resultSet = mock[ResultSet]
166 | val schema = mock[Schema]
167 | val field = mock[Field]
168 | val struct = mock[Struct]
169 |
170 | when(schema.field("id")).thenReturn(field)
171 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA)
172 | when(resultSet.next()).thenReturn(true, true, false)
173 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
174 | when(struct.getInt32("id")).thenReturn(4, 5)
175 | when(struct.get("time")).thenReturn(new Date(3L), new Date(3L))
176 |
177 | timeIdBasedDataServiceWithLargerOffsetMssql.extractRecords(resultSet, schema).toString shouldBe
178 | Success(
179 | ListBuffer(
180 | new SourceRecord(
181 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
182 | Map(TimestampMode.entryName -> 3L, IncrementingMode.entryName -> 4).asJava,
183 | "time-id-based-data-topic", schema, struct
184 | ),
185 | new SourceRecord(
186 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
187 | Map(TimestampMode.entryName -> 3L, IncrementingMode.entryName -> 5).asJava,
188 | "time-id-based-data-topic", schema, struct
189 | )
190 | )
191 | ).toString
192 |
193 | timeIdBasedDataServiceWithLargerOffsetMssql.timestampOffset shouldBe 3L
194 | timeIdBasedDataServiceWithLargerOffsetMssql.incrementingOffset shouldBe 5L
195 | }
196 |
197 | "extract records with key" in {
198 | val timeIdBasedDataServiceWithKeyMysql = timeIdBasedDataServiceMysql.copy(keyFieldOpt = Some("key"))
199 | val resultSet = mock[ResultSet]
200 | val schema = mock[Schema]
201 | val field = mock[Field]
202 | val struct = mock[Struct]
203 |
204 | when(schema.field("id")).thenReturn(field)
205 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA)
206 | when(resultSet.next()).thenReturn(true, true, false)
207 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
208 | when(struct.getInt32("id")).thenReturn(1, 2)
209 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L))
210 | when(struct.get("key")).thenReturn("key-1", "key-2")
211 |
212 | timeIdBasedDataServiceWithKeyMysql.extractRecords(resultSet, schema).toString shouldBe
213 | Success(
214 | ListBuffer(
215 | new SourceRecord(
216 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
217 | Map(TimestampMode.entryName -> 1L, IncrementingMode.entryName -> 1).asJava,
218 | "time-id-based-data-topic", null, schema, "key-1", schema, struct
219 | ),
220 | new SourceRecord(
221 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava,
222 | Map(TimestampMode.entryName -> 2L, IncrementingMode.entryName -> 2).asJava,
223 | "time-id-based-data-topic", null, schema, "key-2", schema, struct
224 | )
225 | )
226 | ).toString
227 | }
228 |
229 | "returns no record when id field is not of type integer" in {
230 | val resultSet = mock[ResultSet]
231 | val schema = mock[Schema]
232 | val field = mock[Field]
233 | val struct = mock[Struct]
234 |
235 | when(schema.field("id")).thenReturn(field)
236 | when(field.schema()).thenReturn(Schema.STRING_SCHEMA)
237 | when(resultSet.next()).thenReturn(true, false)
238 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct))
239 | when(struct.get("time")).thenReturn(new Date(1L), Nil)
240 |
241 | timeIdBasedDataServiceMysql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer())
242 | }
243 | }
244 | }
245 |
--------------------------------------------------------------------------------
/src/test/scala/com/agoda/kafka/connector/jdbc/utils/DataConverterTest.scala:
--------------------------------------------------------------------------------
1 | package com.agoda.kafka.connector.jdbc.utils
2 |
3 | import java.net.URL
4 | import java.math.BigDecimal
5 | import java.sql.{Blob, Clob, NClob, ResultSet, ResultSetMetaData, SQLXML, Types, Date => SqlDate, Time => SqlTime, Timestamp => SqlTimestamp}
6 |
7 | import org.apache.kafka.connect.data._
8 | import org.apache.kafka.connect.errors.DataException
9 | import org.mockito.Mockito._
10 | import org.mockito.ArgumentMatchers._
11 | import org.scalatest.mockito.MockitoSugar
12 | import org.scalatest.{Matchers, WordSpec}
13 |
14 | import scala.util.{Failure, Success}
15 |
16 | class DataConverterTest extends WordSpec with Matchers with MockitoSugar {
17 |
18 | "Data Converter" should {
19 |
20 | val dataConverter = new DataConverter
21 |
22 | "create schema from result set metadata with non optional columns" in {
23 | val metaRS = mock[ResultSetMetaData]
24 |
25 | when(metaRS.getColumnCount).thenReturn(28)
26 |
27 | when(metaRS.getColumnName(1)).thenReturn("BOOLEAN")
28 | when(metaRS.getColumnName(2)).thenReturn("BIT")
29 | when(metaRS.getColumnName(3)).thenReturn("TINYINT")
30 | when(metaRS.getColumnName(4)).thenReturn("SMALLINT")
31 | when(metaRS.getColumnName(5)).thenReturn("INTEGER")
32 | when(metaRS.getColumnName(6)).thenReturn("BIGINT")
33 | when(metaRS.getColumnName(7)).thenReturn("REAL")
34 | when(metaRS.getColumnName(8)).thenReturn("FLOAT")
35 | when(metaRS.getColumnName(9)).thenReturn("DOUBLE")
36 | when(metaRS.getColumnName(10)).thenReturn("NUMERIC")
37 | when(metaRS.getColumnName(11)).thenReturn("DECIMAL")
38 | when(metaRS.getColumnName(12)).thenReturn("CHAR")
39 | when(metaRS.getColumnName(13)).thenReturn("VARCHAR")
40 | when(metaRS.getColumnName(14)).thenReturn("LONGVARCHAR")
41 | when(metaRS.getColumnName(15)).thenReturn("NCHAR")
42 | when(metaRS.getColumnName(16)).thenReturn("NVARCHAR")
43 | when(metaRS.getColumnName(17)).thenReturn("LONGNVARCHAR")
44 | when(metaRS.getColumnName(18)).thenReturn("CLOB")
45 | when(metaRS.getColumnName(19)).thenReturn("NCLOB")
46 | when(metaRS.getColumnName(20)).thenReturn("DATALINK")
47 | when(metaRS.getColumnName(21)).thenReturn("SQLXML")
48 | when(metaRS.getColumnName(22)).thenReturn("BINARY")
49 | when(metaRS.getColumnName(23)).thenReturn("BLOB")
50 | when(metaRS.getColumnName(24)).thenReturn("VARBINARY")
51 | when(metaRS.getColumnName(25)).thenReturn("LONGVARBINARY")
52 | when(metaRS.getColumnName(26)).thenReturn("DATE")
53 | when(metaRS.getColumnName(27)).thenReturn("TIME")
54 | when(metaRS.getColumnName(28)).thenReturn("TIMESTAMP")
55 |
56 | when(metaRS.getColumnLabel(1)).thenReturn("BOOLEAN")
57 | when(metaRS.getColumnLabel(2)).thenReturn("BIT")
58 | when(metaRS.getColumnLabel(3)).thenReturn("TINYINT")
59 | when(metaRS.getColumnLabel(4)).thenReturn("SMALLINT")
60 | when(metaRS.getColumnLabel(5)).thenReturn("INTEGER")
61 | when(metaRS.getColumnLabel(6)).thenReturn("BIGINT")
62 | when(metaRS.getColumnLabel(7)).thenReturn("REAL")
63 | when(metaRS.getColumnLabel(8)).thenReturn("FLOAT")
64 | when(metaRS.getColumnLabel(9)).thenReturn("DOUBLE")
65 | when(metaRS.getColumnLabel(10)).thenReturn("NUMERIC")
66 | when(metaRS.getColumnLabel(11)).thenReturn("DECIMAL")
67 | when(metaRS.getColumnLabel(12)).thenReturn("CHAR")
68 | when(metaRS.getColumnLabel(13)).thenReturn("VARCHAR")
69 | when(metaRS.getColumnLabel(14)).thenReturn("LONGVARCHAR")
70 | when(metaRS.getColumnLabel(15)).thenReturn("NCHAR")
71 | when(metaRS.getColumnLabel(16)).thenReturn("NVARCHAR")
72 | when(metaRS.getColumnLabel(17)).thenReturn("LONGNVARCHAR")
73 | when(metaRS.getColumnLabel(18)).thenReturn("CLOB")
74 | when(metaRS.getColumnLabel(19)).thenReturn("NCLOB")
75 | when(metaRS.getColumnLabel(20)).thenReturn("DATALINK")
76 | when(metaRS.getColumnLabel(21)).thenReturn("SQLXML")
77 | when(metaRS.getColumnLabel(22)).thenReturn("BINARY")
78 | when(metaRS.getColumnLabel(23)).thenReturn("BLOB")
79 | when(metaRS.getColumnLabel(24)).thenReturn("VARBINARY")
80 | when(metaRS.getColumnLabel(25)).thenReturn("LONGVARBINARY")
81 | when(metaRS.getColumnLabel(26)).thenReturn("DATE")
82 | when(metaRS.getColumnLabel(27)).thenReturn("TIME")
83 | when(metaRS.getColumnLabel(28)).thenReturn("TIMESTAMP")
84 |
85 | when(metaRS.getColumnType(1)).thenReturn(Types.BOOLEAN)
86 | when(metaRS.getColumnType(2)).thenReturn(Types.BIT)
87 | when(metaRS.getColumnType(3)).thenReturn(Types.TINYINT)
88 | when(metaRS.getColumnType(4)).thenReturn(Types.SMALLINT)
89 | when(metaRS.getColumnType(5)).thenReturn(Types.INTEGER)
90 | when(metaRS.getColumnType(6)).thenReturn(Types.BIGINT)
91 | when(metaRS.getColumnType(7)).thenReturn(Types.REAL)
92 | when(metaRS.getColumnType(8)).thenReturn(Types.FLOAT)
93 | when(metaRS.getColumnType(9)).thenReturn(Types.DOUBLE)
94 | when(metaRS.getColumnType(10)).thenReturn(Types.NUMERIC)
95 | when(metaRS.getColumnType(11)).thenReturn(Types.DECIMAL)
96 | when(metaRS.getColumnType(12)).thenReturn(Types.CHAR)
97 | when(metaRS.getColumnType(13)).thenReturn(Types.VARCHAR)
98 | when(metaRS.getColumnType(14)).thenReturn(Types.LONGVARCHAR)
99 | when(metaRS.getColumnType(15)).thenReturn(Types.NCHAR)
100 | when(metaRS.getColumnType(16)).thenReturn(Types.NVARCHAR)
101 | when(metaRS.getColumnType(17)).thenReturn(Types.LONGNVARCHAR)
102 | when(metaRS.getColumnType(18)).thenReturn(Types.CLOB)
103 | when(metaRS.getColumnType(19)).thenReturn(Types.NCLOB)
104 | when(metaRS.getColumnType(20)).thenReturn(Types.DATALINK)
105 | when(metaRS.getColumnType(21)).thenReturn(Types.SQLXML)
106 | when(metaRS.getColumnType(22)).thenReturn(Types.BINARY)
107 | when(metaRS.getColumnType(23)).thenReturn(Types.BLOB)
108 | when(metaRS.getColumnType(24)).thenReturn(Types.VARBINARY)
109 | when(metaRS.getColumnType(25)).thenReturn(Types.LONGVARBINARY)
110 | when(metaRS.getColumnType(26)).thenReturn(Types.DATE)
111 | when(metaRS.getColumnType(27)).thenReturn(Types.TIME)
112 | when(metaRS.getColumnType(28)).thenReturn(Types.TIMESTAMP)
113 |
114 | when(metaRS.isNullable(1)).thenReturn(ResultSetMetaData.columnNoNulls)
115 | when(metaRS.isNullable(2)).thenReturn(ResultSetMetaData.columnNoNulls)
116 | when(metaRS.isNullable(3)).thenReturn(ResultSetMetaData.columnNoNulls)
117 | when(metaRS.isNullable(4)).thenReturn(ResultSetMetaData.columnNoNulls)
118 | when(metaRS.isNullable(5)).thenReturn(ResultSetMetaData.columnNoNulls)
119 | when(metaRS.isNullable(6)).thenReturn(ResultSetMetaData.columnNoNulls)
120 | when(metaRS.isNullable(7)).thenReturn(ResultSetMetaData.columnNoNulls)
121 | when(metaRS.isNullable(8)).thenReturn(ResultSetMetaData.columnNoNulls)
122 | when(metaRS.isNullable(9)).thenReturn(ResultSetMetaData.columnNoNulls)
123 | when(metaRS.isNullable(10)).thenReturn(ResultSetMetaData.columnNoNulls)
124 | when(metaRS.isNullable(11)).thenReturn(ResultSetMetaData.columnNoNulls)
125 | when(metaRS.isNullable(12)).thenReturn(ResultSetMetaData.columnNoNulls)
126 | when(metaRS.isNullable(13)).thenReturn(ResultSetMetaData.columnNoNulls)
127 | when(metaRS.isNullable(14)).thenReturn(ResultSetMetaData.columnNoNulls)
128 | when(metaRS.isNullable(15)).thenReturn(ResultSetMetaData.columnNoNulls)
129 | when(metaRS.isNullable(16)).thenReturn(ResultSetMetaData.columnNoNulls)
130 | when(metaRS.isNullable(17)).thenReturn(ResultSetMetaData.columnNoNulls)
131 | when(metaRS.isNullable(18)).thenReturn(ResultSetMetaData.columnNoNulls)
132 | when(metaRS.isNullable(19)).thenReturn(ResultSetMetaData.columnNoNulls)
133 | when(metaRS.isNullable(20)).thenReturn(ResultSetMetaData.columnNoNulls)
134 | when(metaRS.isNullable(21)).thenReturn(ResultSetMetaData.columnNoNulls)
135 | when(metaRS.isNullable(22)).thenReturn(ResultSetMetaData.columnNoNulls)
136 | when(metaRS.isNullable(23)).thenReturn(ResultSetMetaData.columnNoNulls)
137 | when(metaRS.isNullable(24)).thenReturn(ResultSetMetaData.columnNoNulls)
138 | when(metaRS.isNullable(25)).thenReturn(ResultSetMetaData.columnNoNulls)
139 | when(metaRS.isNullable(26)).thenReturn(ResultSetMetaData.columnNoNulls)
140 | when(metaRS.isNullable(27)).thenReturn(ResultSetMetaData.columnNoNulls)
141 | when(metaRS.isNullable(28)).thenReturn(ResultSetMetaData.columnNoNulls)
142 |
143 | when(metaRS.getScale(10)).thenReturn(2)
144 | when(metaRS.getScale(11)).thenReturn(3)
145 |
146 | val s = dataConverter.convertSchema("test-procedure", metaRS)
147 |
148 | s.map(_.name) shouldBe Success("test-procedure")
149 | s.map(_.field("BOOLEAN").schema()) shouldBe Success(Schema.BOOLEAN_SCHEMA)
150 | s.map(_.field("BIT").schema()) shouldBe Success(Schema.INT8_SCHEMA)
151 | s.map(_.field("TINYINT").schema()) shouldBe Success(Schema.INT8_SCHEMA)
152 | s.map(_.field("SMALLINT").schema()) shouldBe Success(Schema.INT16_SCHEMA)
153 | s.map(_.field("INTEGER").schema()) shouldBe Success(Schema.INT32_SCHEMA)
154 | s.map(_.field("BIGINT").schema()) shouldBe Success(Schema.INT64_SCHEMA)
155 | s.map(_.field("REAL").schema()) shouldBe Success(Schema.FLOAT32_SCHEMA)
156 | s.map(_.field("FLOAT").schema()) shouldBe Success(Schema.FLOAT64_SCHEMA)
157 | s.map(_.field("DOUBLE").schema()) shouldBe Success(Schema.FLOAT64_SCHEMA)
158 | s.map(_.field("NUMERIC").schema()) shouldBe Success(Decimal.builder(2).build())
159 | s.map(_.field("DECIMAL").schema()) shouldBe Success(Decimal.builder(3).build())
160 | s.map(_.field("CHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA)
161 | s.map(_.field("VARCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA)
162 | s.map(_.field("LONGVARCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA)
163 | s.map(_.field("NCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA)
164 | s.map(_.field("NVARCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA)
165 | s.map(_.field("LONGNVARCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA)
166 | s.map(_.field("CLOB").schema()) shouldBe Success(Schema.STRING_SCHEMA)
167 | s.map(_.field("NCLOB").schema()) shouldBe Success(Schema.STRING_SCHEMA)
168 | s.map(_.field("DATALINK").schema()) shouldBe Success(Schema.STRING_SCHEMA)
169 | s.map(_.field("SQLXML").schema()) shouldBe Success(Schema.STRING_SCHEMA)
170 | s.map(_.field("BINARY").schema()) shouldBe Success(Schema.BYTES_SCHEMA)
171 | s.map(_.field("BLOB").schema()) shouldBe Success(Schema.BYTES_SCHEMA)
172 | s.map(_.field("VARBINARY").schema()) shouldBe Success(Schema.BYTES_SCHEMA)
173 | s.map(_.field("LONGVARBINARY").schema()) shouldBe Success(Schema.BYTES_SCHEMA)
174 | s.map(_.field("DATE").schema()) shouldBe Success(Date.builder().build())
175 | s.map(_.field("TIME").schema()) shouldBe Success(Time.builder().build())
176 | s.map(_.field("TIMESTAMP").schema()) shouldBe Success(Timestamp.builder().build())
177 | }
178 |
179 | "create schema from result set metadata with optional columns" in {
180 | val metaRS = mock[ResultSetMetaData]
181 |
182 | when(metaRS.getColumnCount).thenReturn(28)
183 |
184 | when(metaRS.getColumnName(1)).thenReturn("BOOLEAN")
185 | when(metaRS.getColumnName(2)).thenReturn("BIT")
186 | when(metaRS.getColumnName(3)).thenReturn("TINYINT")
187 | when(metaRS.getColumnName(4)).thenReturn("SMALLINT")
188 | when(metaRS.getColumnName(5)).thenReturn("INTEGER")
189 | when(metaRS.getColumnName(6)).thenReturn("BIGINT")
190 | when(metaRS.getColumnName(7)).thenReturn("REAL")
191 | when(metaRS.getColumnName(8)).thenReturn("FLOAT")
192 | when(metaRS.getColumnName(9)).thenReturn("DOUBLE")
193 | when(metaRS.getColumnName(10)).thenReturn("NUMERIC")
194 | when(metaRS.getColumnName(11)).thenReturn("DECIMAL")
195 | when(metaRS.getColumnName(12)).thenReturn("CHAR")
196 | when(metaRS.getColumnName(13)).thenReturn("VARCHAR")
197 | when(metaRS.getColumnName(14)).thenReturn("LONGVARCHAR")
198 | when(metaRS.getColumnName(15)).thenReturn("NCHAR")
199 | when(metaRS.getColumnName(16)).thenReturn("NVARCHAR")
200 | when(metaRS.getColumnName(17)).thenReturn("LONGNVARCHAR")
201 | when(metaRS.getColumnName(18)).thenReturn("CLOB")
202 | when(metaRS.getColumnName(19)).thenReturn("NCLOB")
203 | when(metaRS.getColumnName(20)).thenReturn("DATALINK")
204 | when(metaRS.getColumnName(21)).thenReturn("SQLXML")
205 | when(metaRS.getColumnName(22)).thenReturn("BINARY")
206 | when(metaRS.getColumnName(23)).thenReturn("BLOB")
207 | when(metaRS.getColumnName(24)).thenReturn("VARBINARY")
208 | when(metaRS.getColumnName(25)).thenReturn("LONGVARBINARY")
209 | when(metaRS.getColumnName(26)).thenReturn("DATE")
210 | when(metaRS.getColumnName(27)).thenReturn("TIME")
211 | when(metaRS.getColumnName(28)).thenReturn("TIMESTAMP")
212 |
213 | when(metaRS.getColumnLabel(1)).thenReturn("BOOLEAN")
214 | when(metaRS.getColumnLabel(2)).thenReturn("BIT")
215 | when(metaRS.getColumnLabel(3)).thenReturn("TINYINT")
216 | when(metaRS.getColumnLabel(4)).thenReturn("SMALLINT")
217 | when(metaRS.getColumnLabel(5)).thenReturn("INTEGER")
218 | when(metaRS.getColumnLabel(6)).thenReturn("BIGINT")
219 | when(metaRS.getColumnLabel(7)).thenReturn("REAL")
220 | when(metaRS.getColumnLabel(8)).thenReturn("FLOAT")
221 | when(metaRS.getColumnLabel(9)).thenReturn("DOUBLE")
222 | when(metaRS.getColumnLabel(10)).thenReturn("NUMERIC")
223 | when(metaRS.getColumnLabel(11)).thenReturn("DECIMAL")
224 | when(metaRS.getColumnLabel(12)).thenReturn("CHAR")
225 | when(metaRS.getColumnLabel(13)).thenReturn("VARCHAR")
226 | when(metaRS.getColumnLabel(14)).thenReturn("LONGVARCHAR")
227 | when(metaRS.getColumnLabel(15)).thenReturn("NCHAR")
228 | when(metaRS.getColumnLabel(16)).thenReturn("NVARCHAR")
229 | when(metaRS.getColumnLabel(17)).thenReturn("LONGNVARCHAR")
230 | when(metaRS.getColumnLabel(18)).thenReturn("CLOB")
231 | when(metaRS.getColumnLabel(19)).thenReturn("NCLOB")
232 | when(metaRS.getColumnLabel(20)).thenReturn("DATALINK")
233 | when(metaRS.getColumnLabel(21)).thenReturn("SQLXML")
234 | when(metaRS.getColumnLabel(22)).thenReturn("BINARY")
235 | when(metaRS.getColumnLabel(23)).thenReturn("BLOB")
236 | when(metaRS.getColumnLabel(24)).thenReturn("VARBINARY")
237 | when(metaRS.getColumnLabel(25)).thenReturn("LONGVARBINARY")
238 | when(metaRS.getColumnLabel(26)).thenReturn("DATE")
239 | when(metaRS.getColumnLabel(27)).thenReturn("TIME")
240 | when(metaRS.getColumnLabel(28)).thenReturn("TIMESTAMP")
241 |
242 | when(metaRS.getColumnType(1)).thenReturn(Types.BOOLEAN)
243 | when(metaRS.getColumnType(2)).thenReturn(Types.BIT)
244 | when(metaRS.getColumnType(3)).thenReturn(Types.TINYINT)
245 | when(metaRS.getColumnType(4)).thenReturn(Types.SMALLINT)
246 | when(metaRS.getColumnType(5)).thenReturn(Types.INTEGER)
247 | when(metaRS.getColumnType(6)).thenReturn(Types.BIGINT)
248 | when(metaRS.getColumnType(7)).thenReturn(Types.REAL)
249 | when(metaRS.getColumnType(8)).thenReturn(Types.FLOAT)
250 | when(metaRS.getColumnType(9)).thenReturn(Types.DOUBLE)
251 | when(metaRS.getColumnType(10)).thenReturn(Types.NUMERIC)
252 | when(metaRS.getColumnType(11)).thenReturn(Types.DECIMAL)
253 | when(metaRS.getColumnType(12)).thenReturn(Types.CHAR)
254 | when(metaRS.getColumnType(13)).thenReturn(Types.VARCHAR)
255 | when(metaRS.getColumnType(14)).thenReturn(Types.LONGVARCHAR)
256 | when(metaRS.getColumnType(15)).thenReturn(Types.NCHAR)
257 | when(metaRS.getColumnType(16)).thenReturn(Types.NVARCHAR)
258 | when(metaRS.getColumnType(17)).thenReturn(Types.LONGNVARCHAR)
259 | when(metaRS.getColumnType(18)).thenReturn(Types.CLOB)
260 | when(metaRS.getColumnType(19)).thenReturn(Types.NCLOB)
261 | when(metaRS.getColumnType(20)).thenReturn(Types.DATALINK)
262 | when(metaRS.getColumnType(21)).thenReturn(Types.SQLXML)
263 | when(metaRS.getColumnType(22)).thenReturn(Types.BINARY)
264 | when(metaRS.getColumnType(23)).thenReturn(Types.BLOB)
265 | when(metaRS.getColumnType(24)).thenReturn(Types.VARBINARY)
266 | when(metaRS.getColumnType(25)).thenReturn(Types.LONGVARBINARY)
267 | when(metaRS.getColumnType(26)).thenReturn(Types.DATE)
268 | when(metaRS.getColumnType(27)).thenReturn(Types.TIME)
269 | when(metaRS.getColumnType(28)).thenReturn(Types.TIMESTAMP)
270 |
271 | when(metaRS.isNullable(1)).thenReturn(ResultSetMetaData.columnNullable)
272 | when(metaRS.isNullable(2)).thenReturn(ResultSetMetaData.columnNullable)
273 | when(metaRS.isNullable(3)).thenReturn(ResultSetMetaData.columnNullable)
274 | when(metaRS.isNullable(4)).thenReturn(ResultSetMetaData.columnNullable)
275 | when(metaRS.isNullable(5)).thenReturn(ResultSetMetaData.columnNullable)
276 | when(metaRS.isNullable(6)).thenReturn(ResultSetMetaData.columnNullable)
277 | when(metaRS.isNullable(7)).thenReturn(ResultSetMetaData.columnNullable)
278 | when(metaRS.isNullable(8)).thenReturn(ResultSetMetaData.columnNullable)
279 | when(metaRS.isNullable(9)).thenReturn(ResultSetMetaData.columnNullable)
280 | when(metaRS.isNullable(10)).thenReturn(ResultSetMetaData.columnNullable)
281 | when(metaRS.isNullable(11)).thenReturn(ResultSetMetaData.columnNullable)
282 | when(metaRS.isNullable(12)).thenReturn(ResultSetMetaData.columnNullable)
283 | when(metaRS.isNullable(13)).thenReturn(ResultSetMetaData.columnNullable)
284 | when(metaRS.isNullable(14)).thenReturn(ResultSetMetaData.columnNullable)
285 | when(metaRS.isNullable(15)).thenReturn(ResultSetMetaData.columnNullable)
286 | when(metaRS.isNullable(16)).thenReturn(ResultSetMetaData.columnNullable)
287 | when(metaRS.isNullable(17)).thenReturn(ResultSetMetaData.columnNullable)
288 | when(metaRS.isNullable(18)).thenReturn(ResultSetMetaData.columnNullable)
289 | when(metaRS.isNullable(19)).thenReturn(ResultSetMetaData.columnNullable)
290 | when(metaRS.isNullable(20)).thenReturn(ResultSetMetaData.columnNullable)
291 | when(metaRS.isNullable(21)).thenReturn(ResultSetMetaData.columnNullable)
292 | when(metaRS.isNullable(22)).thenReturn(ResultSetMetaData.columnNullable)
293 | when(metaRS.isNullable(23)).thenReturn(ResultSetMetaData.columnNullable)
294 | when(metaRS.isNullable(24)).thenReturn(ResultSetMetaData.columnNullable)
295 | when(metaRS.isNullable(25)).thenReturn(ResultSetMetaData.columnNullable)
296 | when(metaRS.isNullable(26)).thenReturn(ResultSetMetaData.columnNullable)
297 | when(metaRS.isNullable(27)).thenReturn(ResultSetMetaData.columnNullable)
298 | when(metaRS.isNullable(28)).thenReturn(ResultSetMetaData.columnNullable)
299 |
300 | when(metaRS.getScale(10)).thenReturn(2)
301 | when(metaRS.getScale(11)).thenReturn(3)
302 |
303 | val s = dataConverter.convertSchema("test-procedure", metaRS)
304 |
305 | s.map(_.name) shouldBe Success("test-procedure")
306 | s.map(_.field("BOOLEAN").schema()) shouldBe Success(Schema.OPTIONAL_BOOLEAN_SCHEMA)
307 | s.map(_.field("BIT").schema()) shouldBe Success(Schema.OPTIONAL_INT8_SCHEMA)
308 | s.map(_.field("TINYINT").schema()) shouldBe Success(Schema.OPTIONAL_INT8_SCHEMA)
309 | s.map(_.field("SMALLINT").schema()) shouldBe Success(Schema.OPTIONAL_INT16_SCHEMA)
310 | s.map(_.field("INTEGER").schema()) shouldBe Success(Schema.OPTIONAL_INT32_SCHEMA)
311 | s.map(_.field("BIGINT").schema()) shouldBe Success(Schema.OPTIONAL_INT64_SCHEMA)
312 | s.map(_.field("REAL").schema()) shouldBe Success(Schema.OPTIONAL_FLOAT32_SCHEMA)
313 | s.map(_.field("FLOAT").schema()) shouldBe Success(Schema.OPTIONAL_FLOAT64_SCHEMA)
314 | s.map(_.field("DOUBLE").schema()) shouldBe Success(Schema.OPTIONAL_FLOAT64_SCHEMA)
315 | s.map(_.field("NUMERIC").schema()) shouldBe Success(Decimal.builder(2).optional().build())
316 | s.map(_.field("DECIMAL").schema()) shouldBe Success(Decimal.builder(3).optional().build())
317 | s.map(_.field("CHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
318 | s.map(_.field("VARCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
319 | s.map(_.field("LONGVARCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
320 | s.map(_.field("NCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
321 | s.map(_.field("NVARCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
322 | s.map(_.field("LONGNVARCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
323 | s.map(_.field("CLOB").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
324 | s.map(_.field("NCLOB").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
325 | s.map(_.field("DATALINK").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
326 | s.map(_.field("SQLXML").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA)
327 | s.map(_.field("BINARY").schema()) shouldBe Success(Schema.OPTIONAL_BYTES_SCHEMA)
328 | s.map(_.field("BLOB").schema()) shouldBe Success(Schema.OPTIONAL_BYTES_SCHEMA)
329 | s.map(_.field("VARBINARY").schema()) shouldBe Success(Schema.OPTIONAL_BYTES_SCHEMA)
330 | s.map(_.field("LONGVARBINARY").schema()) shouldBe Success(Schema.OPTIONAL_BYTES_SCHEMA)
331 | s.map(_.field("DATE").schema()) shouldBe Success(Date.builder().optional().build())
332 | s.map(_.field("TIME").schema()) shouldBe Success(Time.builder().optional().build())
333 | s.map(_.field("TIMESTAMP").schema()) shouldBe Success(Timestamp.builder().optional().build())
334 | }
335 |
336 | "skip unsupported SQL types while creating schema" in {
337 | val metaRS = mock[ResultSetMetaData]
338 |
339 | when(metaRS.getColumnCount).thenReturn(2)
340 | when(metaRS.getColumnName(1)).thenReturn("KEY")
341 | when(metaRS.getColumnName(2)).thenReturn("VALUE")
342 | when(metaRS.getColumnLabel(1)).thenReturn(null)
343 | when(metaRS.getColumnLabel(2)).thenReturn("")
344 | when(metaRS.getColumnType(1)).thenReturn(Types.INTEGER)
345 | when(metaRS.getColumnType(2)).thenReturn(Types.JAVA_OBJECT)
346 | when(metaRS.isNullable(1)).thenReturn(ResultSetMetaData.columnNoNulls)
347 | when(metaRS.isNullable(2)).thenReturn(ResultSetMetaData.columnNullableUnknown)
348 |
349 | val s = dataConverter.convertSchema("test-procedure-unsupported", metaRS)
350 |
351 | s.map(_.name) shouldBe Success("test-procedure-unsupported")
352 | s.map(_.fields.size) shouldBe Success(1)
353 | s.map(_.field("KEY").schema()) shouldBe Success(Schema.INT32_SCHEMA)
354 | }
355 |
356 | "convert record from result set with non optional columns" in {
357 | val rS = mock[ResultSet]
358 | val metaRS = mock[ResultSetMetaData]
359 | val blob = mock[Blob]
360 | val clob = mock[Clob]
361 | val nClob = mock[NClob]
362 | val sqlXml = mock[SQLXML]
363 |
364 | val builder = SchemaBuilder.struct().name("test")
365 | builder.field("BOOLEAN", Schema.BOOLEAN_SCHEMA)
366 | builder.field("BIT", Schema.INT8_SCHEMA)
367 | builder.field("TINYINT", Schema.INT8_SCHEMA)
368 | builder.field("SMALLINT", Schema.INT16_SCHEMA)
369 | builder.field("INTEGER", Schema.INT32_SCHEMA)
370 | builder.field("BIGINT", Schema.INT64_SCHEMA)
371 | builder.field("REAL", Schema.FLOAT32_SCHEMA)
372 | builder.field("FLOAT", Schema.FLOAT64_SCHEMA)
373 | builder.field("DOUBLE", Schema.FLOAT64_SCHEMA)
374 | builder.field("NUMERIC", Decimal.builder(1).build())
375 | builder.field("DECIMAL", Decimal.builder(1).build())
376 | builder.field("CHAR", Schema.STRING_SCHEMA)
377 | builder.field("VARCHAR", Schema.STRING_SCHEMA)
378 | builder.field("LONGVARCHAR", Schema.STRING_SCHEMA)
379 | builder.field("NCHAR", Schema.STRING_SCHEMA)
380 | builder.field("NVARCHAR", Schema.STRING_SCHEMA)
381 | builder.field("LONGNVARCHAR", Schema.STRING_SCHEMA)
382 | builder.field("CLOB", Schema.STRING_SCHEMA)
383 | builder.field("NCLOB", Schema.STRING_SCHEMA)
384 | builder.field("DATALINK", Schema.STRING_SCHEMA)
385 | builder.field("SQLXML", Schema.STRING_SCHEMA)
386 | builder.field("BINARY", Schema.BYTES_SCHEMA)
387 | builder.field("VARBINARY", Schema.BYTES_SCHEMA)
388 | builder.field("LONGVARBINARY", Schema.BYTES_SCHEMA)
389 | builder.field("BLOB", Schema.BYTES_SCHEMA)
390 | builder.field("DATE", Date.builder().build())
391 | builder.field("TIME", Time.builder().build())
392 | builder.field("TIMESTAMP", Timestamp.builder().build())
393 | val schema = builder.build()
394 |
395 | when(rS.wasNull()).thenReturn(false)
396 | when(rS.getMetaData).thenReturn(metaRS)
397 | when(metaRS.getColumnCount).thenReturn(28)
398 |
399 | when(metaRS.getColumnName(1)).thenReturn("BOOLEAN")
400 | when(metaRS.getColumnName(2)).thenReturn("BIT")
401 | when(metaRS.getColumnName(3)).thenReturn("TINYINT")
402 | when(metaRS.getColumnName(4)).thenReturn("SMALLINT")
403 | when(metaRS.getColumnName(5)).thenReturn("INTEGER")
404 | when(metaRS.getColumnName(6)).thenReturn("BIGINT")
405 | when(metaRS.getColumnName(7)).thenReturn("REAL")
406 | when(metaRS.getColumnName(8)).thenReturn("FLOAT")
407 | when(metaRS.getColumnName(9)).thenReturn("DOUBLE")
408 | when(metaRS.getColumnName(10)).thenReturn("NUMERIC")
409 | when(metaRS.getColumnName(11)).thenReturn("DECIMAL")
410 | when(metaRS.getColumnName(12)).thenReturn("CHAR")
411 | when(metaRS.getColumnName(13)).thenReturn("VARCHAR")
412 | when(metaRS.getColumnName(14)).thenReturn("LONGVARCHAR")
413 | when(metaRS.getColumnName(15)).thenReturn("NCHAR")
414 | when(metaRS.getColumnName(16)).thenReturn("NVARCHAR")
415 | when(metaRS.getColumnName(17)).thenReturn("LONGNVARCHAR")
416 | when(metaRS.getColumnName(18)).thenReturn("CLOB")
417 | when(metaRS.getColumnName(19)).thenReturn("NCLOB")
418 | when(metaRS.getColumnName(20)).thenReturn("DATALINK")
419 | when(metaRS.getColumnName(21)).thenReturn("SQLXML")
420 | when(metaRS.getColumnName(22)).thenReturn("BINARY")
421 | when(metaRS.getColumnName(23)).thenReturn("VARBINARY")
422 | when(metaRS.getColumnName(24)).thenReturn("LONGVARBINARY")
423 | when(metaRS.getColumnName(25)).thenReturn("BLOB")
424 | when(metaRS.getColumnName(26)).thenReturn("DATE")
425 | when(metaRS.getColumnName(27)).thenReturn("TIME")
426 | when(metaRS.getColumnName(28)).thenReturn("TIMESTAMP")
427 |
428 | when(metaRS.getColumnLabel(1)).thenReturn("BOOLEAN")
429 | when(metaRS.getColumnLabel(2)).thenReturn("BIT")
430 | when(metaRS.getColumnLabel(3)).thenReturn("TINYINT")
431 | when(metaRS.getColumnLabel(4)).thenReturn("SMALLINT")
432 | when(metaRS.getColumnLabel(5)).thenReturn("INTEGER")
433 | when(metaRS.getColumnLabel(6)).thenReturn("BIGINT")
434 | when(metaRS.getColumnLabel(7)).thenReturn("REAL")
435 | when(metaRS.getColumnLabel(8)).thenReturn("FLOAT")
436 | when(metaRS.getColumnLabel(9)).thenReturn("DOUBLE")
437 | when(metaRS.getColumnLabel(10)).thenReturn("NUMERIC")
438 | when(metaRS.getColumnLabel(11)).thenReturn("DECIMAL")
439 | when(metaRS.getColumnLabel(12)).thenReturn("CHAR")
440 | when(metaRS.getColumnLabel(13)).thenReturn("VARCHAR")
441 | when(metaRS.getColumnLabel(14)).thenReturn("LONGVARCHAR")
442 | when(metaRS.getColumnLabel(15)).thenReturn("NCHAR")
443 | when(metaRS.getColumnLabel(16)).thenReturn("NVARCHAR")
444 | when(metaRS.getColumnLabel(17)).thenReturn("LONGNVARCHAR")
445 | when(metaRS.getColumnLabel(18)).thenReturn("CLOB")
446 | when(metaRS.getColumnLabel(19)).thenReturn("NCLOB")
447 | when(metaRS.getColumnLabel(20)).thenReturn("DATALINK")
448 | when(metaRS.getColumnLabel(21)).thenReturn("SQLXML")
449 | when(metaRS.getColumnLabel(22)).thenReturn("BINARY")
450 | when(metaRS.getColumnLabel(23)).thenReturn("VARBINARY")
451 | when(metaRS.getColumnLabel(24)).thenReturn("LONGVARBINARY")
452 | when(metaRS.getColumnLabel(25)).thenReturn("BLOB")
453 | when(metaRS.getColumnLabel(26)).thenReturn("DATE")
454 | when(metaRS.getColumnLabel(27)).thenReturn("TIME")
455 | when(metaRS.getColumnLabel(28)).thenReturn("TIMESTAMP")
456 |
457 | when(metaRS.getColumnType(1)).thenReturn(Types.BOOLEAN)
458 | when(metaRS.getColumnType(2)).thenReturn(Types.BIT)
459 | when(metaRS.getColumnType(3)).thenReturn(Types.TINYINT)
460 | when(metaRS.getColumnType(4)).thenReturn(Types.SMALLINT)
461 | when(metaRS.getColumnType(5)).thenReturn(Types.INTEGER)
462 | when(metaRS.getColumnType(6)).thenReturn(Types.BIGINT)
463 | when(metaRS.getColumnType(7)).thenReturn(Types.REAL)
464 | when(metaRS.getColumnType(8)).thenReturn(Types.FLOAT)
465 | when(metaRS.getColumnType(9)).thenReturn(Types.DOUBLE)
466 | when(metaRS.getColumnType(10)).thenReturn(Types.NUMERIC)
467 | when(metaRS.getColumnType(11)).thenReturn(Types.DECIMAL)
468 | when(metaRS.getColumnType(12)).thenReturn(Types.CHAR)
469 | when(metaRS.getColumnType(13)).thenReturn(Types.VARCHAR)
470 | when(metaRS.getColumnType(14)).thenReturn(Types.LONGVARCHAR)
471 | when(metaRS.getColumnType(15)).thenReturn(Types.NCHAR)
472 | when(metaRS.getColumnType(16)).thenReturn(Types.NVARCHAR)
473 | when(metaRS.getColumnType(17)).thenReturn(Types.LONGNVARCHAR)
474 | when(metaRS.getColumnType(18)).thenReturn(Types.CLOB)
475 | when(metaRS.getColumnType(19)).thenReturn(Types.NCLOB)
476 | when(metaRS.getColumnType(20)).thenReturn(Types.DATALINK)
477 | when(metaRS.getColumnType(21)).thenReturn(Types.SQLXML)
478 | when(metaRS.getColumnType(22)).thenReturn(Types.BINARY)
479 | when(metaRS.getColumnType(23)).thenReturn(Types.VARBINARY)
480 | when(metaRS.getColumnType(24)).thenReturn(Types.LONGVARBINARY)
481 | when(metaRS.getColumnType(25)).thenReturn(Types.BLOB)
482 | when(metaRS.getColumnType(26)).thenReturn(Types.DATE)
483 | when(metaRS.getColumnType(27)).thenReturn(Types.TIME)
484 | when(metaRS.getColumnType(28)).thenReturn(Types.TIMESTAMP)
485 |
486 | when(blob.getBytes(1, blob.length.toInt)).thenReturn("25".toCharArray.map(_.toByte))
487 | when(clob.getSubString(1, clob.length.toInt)).thenReturn("18")
488 | when(nClob.getSubString(1, clob.length.toInt)).thenReturn("19")
489 | when(sqlXml.getString).thenReturn("21")
490 |
491 | when(rS.getBoolean(1)).thenReturn(true)
492 | when(rS.getByte(2)).thenReturn(2.toByte)
493 | when(rS.getByte(3)).thenReturn(3.toByte)
494 | when(rS.getShort(4)).thenReturn(4.toShort)
495 | when(rS.getInt(5)).thenReturn(5)
496 | when(rS.getLong(6)).thenReturn(6L)
497 | when(rS.getFloat(7)).thenReturn(7.0F)
498 | when(rS.getDouble(8)).thenReturn(8.0)
499 | when(rS.getDouble(9)).thenReturn(9.0)
500 | when(rS.getBigDecimal(10)).thenReturn(new BigDecimal("10.0"))
501 | when(rS.getBigDecimal(11)).thenReturn(new BigDecimal("11.0"))
502 | when(rS.getString(12)).thenReturn("12")
503 | when(rS.getString(13)).thenReturn("13")
504 | when(rS.getString(14)).thenReturn("14")
505 | when(rS.getNString(15)).thenReturn("15")
506 | when(rS.getNString(16)).thenReturn("16")
507 | when(rS.getNString(17)).thenReturn("17")
508 | when(rS.getClob(18)).thenReturn(clob)
509 | when(rS.getNClob(19)).thenReturn(nClob)
510 | when(rS.getURL(20)).thenReturn(new URL("http://20"))
511 | when(rS.getSQLXML(21)).thenReturn(sqlXml)
512 | when(rS.getBytes(22)).thenReturn("22".toCharArray.map(_.toByte))
513 | when(rS.getBytes(23)).thenReturn("23".toCharArray.map(_.toByte))
514 | when(rS.getBytes(24)).thenReturn("24".toCharArray.map(_.toByte))
515 | when(rS.getBlob(25)).thenReturn(blob)
516 | when(rS.getDate(same(26), any())).thenReturn(new SqlDate(26L))
517 | when(rS.getTime(same(27), any())).thenReturn(new SqlTime(27L))
518 | when(rS.getTimestamp(same(28), any())).thenReturn(new SqlTimestamp(28L))
519 |
520 | val r = dataConverter.convertRecord(schema, rS)
521 |
522 | r.map(_.schema) shouldBe Success(schema)
523 | r.map(_.getBoolean("BOOLEAN")) shouldBe Success(true)
524 | r.map(_.getInt8("BIT")) shouldBe Success(2)
525 | r.map(_.getInt8("TINYINT")) shouldBe Success(3)
526 | r.map(_.getInt16("SMALLINT")) shouldBe Success(4)
527 | r.map(_.getInt32("INTEGER")) shouldBe Success(5)
528 | r.map(_.getInt64("BIGINT")) shouldBe Success(6L)
529 | r.map(_.getFloat32("REAL")) shouldBe Success(7.0F)
530 | r.map(_.getFloat64("FLOAT")) shouldBe Success(8.0F)
531 | r.map(_.getFloat64("DOUBLE")) shouldBe Success(9.0F)
532 | r.map(_.get("NUMERIC")) shouldBe Success(new BigDecimal("10.0"))
533 | r.map(_.get("DECIMAL")) shouldBe Success(new BigDecimal("11.0"))
534 | r.map(_.getString("CHAR")) shouldBe Success("12")
535 | r.map(_.getString("VARCHAR")) shouldBe Success("13")
536 | r.map(_.getString("LONGVARCHAR")) shouldBe Success("14")
537 | r.map(_.getString("NCHAR")) shouldBe Success("15")
538 | r.map(_.getString("NVARCHAR")) shouldBe Success("16")
539 | r.map(_.getString("LONGNVARCHAR")) shouldBe Success("17")
540 | r.map(_.getString("CLOB")) shouldBe Success("18")
541 | r.map(_.getString("NCLOB")) shouldBe Success("19")
542 | r.map(_.getString("DATALINK")) shouldBe Success("http://20")
543 | r.map(_.getString("SQLXML")) shouldBe Success("21")
544 | r.map(_.getBytes("BINARY").toSeq) shouldBe Success(Seq('2', '2'))
545 | r.map(_.getBytes("VARBINARY").toSeq) shouldBe Success(Seq('2', '3'))
546 | r.map(_.getBytes("LONGVARBINARY").toSeq) shouldBe Success(Seq('2', '4'))
547 | r.map(_.getBytes("BLOB").toSeq) shouldBe Success(Seq('2', '5'))
548 | r.map(_.get("DATE")) shouldBe Success(new SqlDate(26L))
549 | r.map(_.get("TIME")) shouldBe Success(new SqlTime(27L))
550 | r.map(_.get("TIMESTAMP")) shouldBe Success(new SqlTimestamp(28L))
551 | }
552 |
553 | "convert record from result set with optional columns" in {
554 | val rS = mock[ResultSet]
555 | val metaRS = mock[ResultSetMetaData]
556 |
557 | val builder = SchemaBuilder.struct().name("test")
558 | builder.field("BOOLEAN", Schema.OPTIONAL_BOOLEAN_SCHEMA)
559 | builder.field("BIT", Schema.OPTIONAL_INT8_SCHEMA)
560 | builder.field("TINYINT", Schema.OPTIONAL_INT8_SCHEMA)
561 | builder.field("SMALLINT", Schema.OPTIONAL_INT16_SCHEMA)
562 | builder.field("INTEGER", Schema.OPTIONAL_INT32_SCHEMA)
563 | builder.field("BIGINT", Schema.OPTIONAL_INT64_SCHEMA)
564 | builder.field("REAL", Schema.OPTIONAL_FLOAT32_SCHEMA)
565 | builder.field("FLOAT", Schema.OPTIONAL_FLOAT64_SCHEMA)
566 | builder.field("DOUBLE", Schema.OPTIONAL_FLOAT64_SCHEMA)
567 | builder.field("NUMERIC", Decimal.builder(1).optional().build())
568 | builder.field("DECIMAL", Decimal.builder(1).optional().build())
569 | builder.field("CHAR", Schema.OPTIONAL_STRING_SCHEMA)
570 | builder.field("VARCHAR", Schema.OPTIONAL_STRING_SCHEMA)
571 | builder.field("LONGVARCHAR", Schema.OPTIONAL_STRING_SCHEMA)
572 | builder.field("NCHAR", Schema.OPTIONAL_STRING_SCHEMA)
573 | builder.field("NVARCHAR", Schema.OPTIONAL_STRING_SCHEMA)
574 | builder.field("LONGNVARCHAR", Schema.OPTIONAL_STRING_SCHEMA)
575 | builder.field("CLOB", Schema.OPTIONAL_STRING_SCHEMA)
576 | builder.field("NCLOB", Schema.OPTIONAL_STRING_SCHEMA)
577 | builder.field("DATALINK", Schema.OPTIONAL_STRING_SCHEMA)
578 | builder.field("SQLXML", Schema.OPTIONAL_STRING_SCHEMA)
579 | builder.field("BINARY", Schema.OPTIONAL_BYTES_SCHEMA)
580 | builder.field("VARBINARY", Schema.OPTIONAL_BYTES_SCHEMA)
581 | builder.field("LONGVARBINARY", Schema.OPTIONAL_BYTES_SCHEMA)
582 | builder.field("BLOB", Schema.OPTIONAL_BYTES_SCHEMA)
583 | builder.field("DATE", Date.builder().optional().build())
584 | builder.field("TIME", Time.builder().optional().build())
585 | builder.field("TIMESTAMP", Timestamp.builder().optional().build())
586 | val schema = builder.build()
587 |
588 | when(rS.wasNull()).thenReturn(false)
589 | when(rS.getMetaData).thenReturn(metaRS)
590 | when(metaRS.getColumnCount).thenReturn(28)
591 |
592 | when(metaRS.getColumnName(1)).thenReturn("BOOLEAN")
593 | when(metaRS.getColumnName(2)).thenReturn("BIT")
594 | when(metaRS.getColumnName(3)).thenReturn("TINYINT")
595 | when(metaRS.getColumnName(4)).thenReturn("SMALLINT")
596 | when(metaRS.getColumnName(5)).thenReturn("INTEGER")
597 | when(metaRS.getColumnName(6)).thenReturn("BIGINT")
598 | when(metaRS.getColumnName(7)).thenReturn("REAL")
599 | when(metaRS.getColumnName(8)).thenReturn("FLOAT")
600 | when(metaRS.getColumnName(9)).thenReturn("DOUBLE")
601 | when(metaRS.getColumnName(10)).thenReturn("NUMERIC")
602 | when(metaRS.getColumnName(11)).thenReturn("DECIMAL")
603 | when(metaRS.getColumnName(12)).thenReturn("CHAR")
604 | when(metaRS.getColumnName(13)).thenReturn("VARCHAR")
605 | when(metaRS.getColumnName(14)).thenReturn("LONGVARCHAR")
606 | when(metaRS.getColumnName(15)).thenReturn("NCHAR")
607 | when(metaRS.getColumnName(16)).thenReturn("NVARCHAR")
608 | when(metaRS.getColumnName(17)).thenReturn("LONGNVARCHAR")
609 | when(metaRS.getColumnName(18)).thenReturn("CLOB")
610 | when(metaRS.getColumnName(19)).thenReturn("NCLOB")
611 | when(metaRS.getColumnName(20)).thenReturn("DATALINK")
612 | when(metaRS.getColumnName(21)).thenReturn("SQLXML")
613 | when(metaRS.getColumnName(22)).thenReturn("BINARY")
614 | when(metaRS.getColumnName(23)).thenReturn("VARBINARY")
615 | when(metaRS.getColumnName(24)).thenReturn("LONGVARBINARY")
616 | when(metaRS.getColumnName(25)).thenReturn("BLOB")
617 | when(metaRS.getColumnName(26)).thenReturn("DATE")
618 | when(metaRS.getColumnName(27)).thenReturn("TIME")
619 | when(metaRS.getColumnName(28)).thenReturn("TIMESTAMP")
620 |
621 | when(metaRS.getColumnLabel(1)).thenReturn("BOOLEAN")
622 | when(metaRS.getColumnLabel(2)).thenReturn("BIT")
623 | when(metaRS.getColumnLabel(3)).thenReturn("TINYINT")
624 | when(metaRS.getColumnLabel(4)).thenReturn("SMALLINT")
625 | when(metaRS.getColumnLabel(5)).thenReturn("INTEGER")
626 | when(metaRS.getColumnLabel(6)).thenReturn("BIGINT")
627 | when(metaRS.getColumnLabel(7)).thenReturn("REAL")
628 | when(metaRS.getColumnLabel(8)).thenReturn("FLOAT")
629 | when(metaRS.getColumnLabel(9)).thenReturn("DOUBLE")
630 | when(metaRS.getColumnLabel(10)).thenReturn("NUMERIC")
631 | when(metaRS.getColumnLabel(11)).thenReturn("DECIMAL")
632 | when(metaRS.getColumnLabel(12)).thenReturn("CHAR")
633 | when(metaRS.getColumnLabel(13)).thenReturn("VARCHAR")
634 | when(metaRS.getColumnLabel(14)).thenReturn("LONGVARCHAR")
635 | when(metaRS.getColumnLabel(15)).thenReturn("NCHAR")
636 | when(metaRS.getColumnLabel(16)).thenReturn("NVARCHAR")
637 | when(metaRS.getColumnLabel(17)).thenReturn("LONGNVARCHAR")
638 | when(metaRS.getColumnLabel(18)).thenReturn("CLOB")
639 | when(metaRS.getColumnLabel(19)).thenReturn("NCLOB")
640 | when(metaRS.getColumnLabel(20)).thenReturn("DATALINK")
641 | when(metaRS.getColumnLabel(21)).thenReturn("SQLXML")
642 | when(metaRS.getColumnLabel(22)).thenReturn("BINARY")
643 | when(metaRS.getColumnLabel(23)).thenReturn("VARBINARY")
644 | when(metaRS.getColumnLabel(24)).thenReturn("LONGVARBINARY")
645 | when(metaRS.getColumnLabel(25)).thenReturn("BLOB")
646 | when(metaRS.getColumnLabel(26)).thenReturn("DATE")
647 | when(metaRS.getColumnLabel(27)).thenReturn("TIME")
648 | when(metaRS.getColumnLabel(28)).thenReturn("TIMESTAMP")
649 |
650 | when(metaRS.getColumnType(1)).thenReturn(Types.BOOLEAN)
651 | when(metaRS.getColumnType(2)).thenReturn(Types.BIT)
652 | when(metaRS.getColumnType(3)).thenReturn(Types.TINYINT)
653 | when(metaRS.getColumnType(4)).thenReturn(Types.SMALLINT)
654 | when(metaRS.getColumnType(5)).thenReturn(Types.INTEGER)
655 | when(metaRS.getColumnType(6)).thenReturn(Types.BIGINT)
656 | when(metaRS.getColumnType(7)).thenReturn(Types.REAL)
657 | when(metaRS.getColumnType(8)).thenReturn(Types.FLOAT)
658 | when(metaRS.getColumnType(9)).thenReturn(Types.DOUBLE)
659 | when(metaRS.getColumnType(10)).thenReturn(Types.NUMERIC)
660 | when(metaRS.getColumnType(11)).thenReturn(Types.DECIMAL)
661 | when(metaRS.getColumnType(12)).thenReturn(Types.CHAR)
662 | when(metaRS.getColumnType(13)).thenReturn(Types.VARCHAR)
663 | when(metaRS.getColumnType(14)).thenReturn(Types.LONGVARCHAR)
664 | when(metaRS.getColumnType(15)).thenReturn(Types.NCHAR)
665 | when(metaRS.getColumnType(16)).thenReturn(Types.NVARCHAR)
666 | when(metaRS.getColumnType(17)).thenReturn(Types.LONGNVARCHAR)
667 | when(metaRS.getColumnType(18)).thenReturn(Types.CLOB)
668 | when(metaRS.getColumnType(19)).thenReturn(Types.NCLOB)
669 | when(metaRS.getColumnType(20)).thenReturn(Types.DATALINK)
670 | when(metaRS.getColumnType(21)).thenReturn(Types.SQLXML)
671 | when(metaRS.getColumnType(22)).thenReturn(Types.BINARY)
672 | when(metaRS.getColumnType(23)).thenReturn(Types.VARBINARY)
673 | when(metaRS.getColumnType(24)).thenReturn(Types.LONGVARBINARY)
674 | when(metaRS.getColumnType(25)).thenReturn(Types.BLOB)
675 | when(metaRS.getColumnType(26)).thenReturn(Types.DATE)
676 | when(metaRS.getColumnType(27)).thenReturn(Types.TIME)
677 | when(metaRS.getColumnType(28)).thenReturn(Types.TIMESTAMP)
678 |
679 | when(rS.getBigDecimal(10)).thenReturn(null)
680 | when(rS.getBigDecimal(11)).thenReturn(null)
681 | when(rS.getString(12)).thenReturn(null)
682 | when(rS.getString(13)).thenReturn(null)
683 | when(rS.getString(14)).thenReturn(null)
684 | when(rS.getNString(15)).thenReturn(null)
685 | when(rS.getNString(16)).thenReturn(null)
686 | when(rS.getNString(17)).thenReturn(null)
687 | when(rS.getClob(18)).thenReturn(null)
688 | when(rS.getNClob(19)).thenReturn(null)
689 | when(rS.getURL(20)).thenReturn(null)
690 | when(rS.getSQLXML(21)).thenReturn(null)
691 | when(rS.getBytes(22)).thenReturn(null)
692 | when(rS.getBytes(23)).thenReturn(null)
693 | when(rS.getBytes(24)).thenReturn(null)
694 | when(rS.getBlob(25)).thenReturn(null)
695 | when(rS.getDate(same(26), any())).thenReturn(null)
696 | when(rS.getTime(same(27), any())).thenReturn(null)
697 | when(rS.getTimestamp(same(28), any())).thenReturn(null)
698 |
699 | val r = dataConverter.convertRecord(schema, rS)
700 |
701 | r.map(_.schema) shouldBe Success(schema)
702 | r.map(_.getBoolean("BOOLEAN")) shouldBe Success(false)
703 | r.map(_.getInt8("BIT")) shouldBe Success(0)
704 | r.map(_.getInt8("TINYINT")) shouldBe Success(0)
705 | r.map(_.getInt16("SMALLINT")) shouldBe Success(0)
706 | r.map(_.getInt32("INTEGER")) shouldBe Success(0)
707 | r.map(_.getInt64("BIGINT")) shouldBe Success(0L)
708 | r.map(_.getFloat32("REAL")) shouldBe Success(0.0F)
709 | r.map(_.getFloat64("FLOAT")) shouldBe Success(0.0F)
710 | r.map(_.getFloat64("DOUBLE")) shouldBe Success(0.0F)
711 | r.map(_.get("NUMERIC")) shouldBe Success(null)
712 | r.map(_.get("DECIMAL")) shouldBe Success(null)
713 | r.map(_.getString("CHAR")) shouldBe Success(null)
714 | r.map(_.getString("VARCHAR")) shouldBe Success(null)
715 | r.map(_.getString("LONGVARCHAR")) shouldBe Success(null)
716 | r.map(_.getString("NCHAR")) shouldBe Success(null)
717 | r.map(_.getString("NVARCHAR")) shouldBe Success(null)
718 | r.map(_.getString("LONGNVARCHAR")) shouldBe Success(null)
719 | r.map(_.getString("CLOB")) shouldBe Success(null)
720 | r.map(_.getString("NCLOB")) shouldBe Success(null)
721 | r.map(_.getString("DATALINK")) shouldBe Success(null)
722 | r.map(_.getString("SQLXML")) shouldBe Success(null)
723 | r.map(_.getBytes("BINARY")) shouldBe Success(null)
724 | r.map(_.getBytes("VARBINARY")) shouldBe Success(null)
725 | r.map(_.getBytes("LONGVARBINARY")) shouldBe Success(null)
726 | r.map(_.getBytes("BLOB")) shouldBe Success(null)
727 | r.map(_.get("DATE")) shouldBe Success(null)
728 | r.map(_.get("TIME")) shouldBe Success(null)
729 | r.map(_.get("TIMESTAMP")) shouldBe Success(null)
730 | }
731 |
732 | "return null for unsupported SQL types while converting record" in {
733 | val rS = mock[ResultSet]
734 | val metaRS = mock[ResultSetMetaData]
735 |
736 | val builder = SchemaBuilder.struct().name("test")
737 | builder.field("KEY", Schema.INT32_SCHEMA)
738 | builder.field("VALUE", Schema.OPTIONAL_STRING_SCHEMA)
739 | val schema = builder.build()
740 |
741 | when(rS.getMetaData).thenReturn(metaRS)
742 | when(metaRS.getColumnCount).thenReturn(2)
743 | when(metaRS.getColumnName(1)).thenReturn("KEY")
744 | when(metaRS.getColumnName(2)).thenReturn("VALUE")
745 | when(metaRS.getColumnLabel(1)).thenReturn("KEY")
746 | when(metaRS.getColumnLabel(2)).thenReturn("VALUE")
747 | when(metaRS.getColumnType(1)).thenReturn(Types.INTEGER)
748 | when(metaRS.getColumnType(2)).thenReturn(Types.NULL)
749 | when(rS.getInt(1)).thenReturn(5)
750 |
751 | val r = dataConverter.convertRecord(schema, rS)
752 |
753 | r.map(_.schema) shouldBe Success(schema)
754 | r.map(_.getInt32("KEY")) shouldBe Success(5)
755 | r.map(_.getString("VALUE")) shouldBe Success(null)
756 | }
757 |
758 | "fails if result set is null" in {
759 | val rS = null
760 |
761 | val builder = SchemaBuilder.struct().name("test")
762 | builder.field("KEY", Schema.INT32_SCHEMA)
763 | builder.field("VALUE", Schema.OPTIONAL_STRING_SCHEMA)
764 | val schema = builder.build()
765 |
766 | val r = dataConverter.convertRecord(schema, rS)
767 |
768 | r.getClass shouldBe Failure(new NullPointerException).getClass
769 | }
770 |
771 | "fail if schema and result set don't match" in {
772 | val rS = mock[ResultSet]
773 | val metaRS = mock[ResultSetMetaData]
774 |
775 | val builder = SchemaBuilder.struct().name("test")
776 | builder.field("KEY", Schema.INT32_SCHEMA)
777 | builder.field("VALUE", Schema.BOOLEAN_SCHEMA)
778 | val schema = builder.build()
779 |
780 | when(rS.getMetaData).thenReturn(metaRS)
781 | when(metaRS.getColumnCount).thenReturn(2)
782 | when(metaRS.getColumnName(1)).thenReturn("KEY")
783 | when(metaRS.getColumnName(2)).thenReturn("VALUE")
784 | when(metaRS.getColumnLabel(1)).thenReturn("KEY")
785 | when(metaRS.getColumnLabel(2)).thenReturn("VALUE")
786 | when(metaRS.getColumnType(1)).thenReturn(Types.INTEGER)
787 | when(metaRS.getColumnType(2)).thenReturn(Types.DOUBLE)
788 | when(rS.getInt(1)).thenReturn(5)
789 | when(rS.getDouble(2)).thenReturn(3.14)
790 |
791 | val r = dataConverter.convertRecord(schema, rS)
792 |
793 | r.isFailure shouldEqual true
794 | the [DataException] thrownBy r.get
795 | }
796 | }
797 | }
798 |
--------------------------------------------------------------------------------