├── .gitignore ├── .travis-jvmopts ├── .travis.yml ├── CHANGE_LOG.md ├── CLA.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── README.md ├── build.sbt ├── codecov.yml ├── logo.png ├── project ├── Dependencies.scala ├── build.properties └── plugins.sbt └── src ├── main └── scala │ └── com │ └── agoda │ └── kafka │ └── connector │ └── jdbc │ ├── JdbcSourceConnector.scala │ ├── JdbcSourceConnectorConfig.scala │ ├── JdbcSourceConnectorConstants.scala │ ├── JdbcSourceTask.scala │ ├── JdbcSourceTaskConfig.scala │ ├── models │ ├── DatabaseProduct.scala │ └── Mode.scala │ ├── services │ ├── DataService.scala │ ├── IdBasedDataService.scala │ ├── TimeBasedDataService.scala │ └── TimeIdBasedDataService.scala │ └── utils │ ├── DataConverter.scala │ └── Version.scala └── test └── scala └── com └── agoda └── kafka └── connector └── jdbc ├── JdbcSourceConnectorConfigTest.scala ├── models ├── DatabaseProductTest.scala └── ModeTest.scala ├── services ├── DataServiceTest.scala ├── IdBasedDataServiceTest.scala ├── TimeBasedDataServiceTest.scala └── TimeIdBasedDataServiceTest.scala └── utils └── DataConverterTest.scala /.gitignore: -------------------------------------------------------------------------------- 1 | # IDE specific 2 | .idea/ 3 | 4 | # SBT specific 5 | target/ -------------------------------------------------------------------------------- /.travis-jvmopts: -------------------------------------------------------------------------------- 1 | # This is used to configure the sbt instance that Travis launches 2 | 3 | -Xms1G 4 | -Xmx1G 5 | -Xss2M 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | sudo: false 3 | scala: 4 | - "2.12.2" 5 | jdk: 6 | - oraclejdk8 7 | env: 8 | matrix: 9 | - SCRIPT="sbt -jvm-opts .travis-jvmopts clean coverage test coverageReport" 10 | # important to use eval, otherwise "&&" is passed as an argument to sbt rather than being processed by bash 11 | script: eval $SCRIPT 12 | cache: 13 | directories: 14 | - $HOME/.ivy2/cache 15 | - $HOME/.sbt/boot 16 | after_success: 17 | - bash <(curl -s https://codecov.io/bash) || echo 'Codecov failed to upload' 18 | notifications: 19 | webhooks: 20 | urls: 21 | - https://webhooks.gitter.im/e/1d93347f19b51af09514 22 | on_success: always # options: [always|never|change] default: always 23 | on_failure: always # options: [always|never|change] default: always 24 | on_start: never # options: [always|never|change] default: always 25 | -------------------------------------------------------------------------------- /CHANGE_LOG.md: -------------------------------------------------------------------------------- 1 | Change Log 2 | ========== 3 | 4 | Release 1.2.0 5 | ------------- 6 | 7 | ### Requirements 8 | 9 | * Scala 2.11.* / 2.12.* 10 | * Kafka 0.9.0.0 11 | * Kafka Connect 0.9.0.0 12 | 13 | ### Supported Databases 14 | 15 | * Mssql 16 | * Mysql 17 | 18 | ### Added 19 | 20 | * Extending support for MySQL database. 21 | 22 | Release 1.0.1 23 | ------------- 24 | 25 | ### Fixed 26 | 27 | * NullPointerException in Kafka source connector while connecting to database. 28 | 29 | Release 1.0.0 30 | ------------- 31 | 32 | ### Requirements 33 | 34 | * Scala 2.11.* / 2.12.* 35 | * Kafka 0.9.0.0 36 | * Kafka Connect 0.9.0.0 37 | 38 | ### Supported Databases 39 | 40 | * Mssql 41 | 42 | ### Added 43 | 44 | * Kafka source connector to capture change data using stored procedures. -------------------------------------------------------------------------------- /CLA.md: -------------------------------------------------------------------------------- 1 | ![Agoda Logo](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/logo.png) 2 | 3 | # INDIVIDUAL CONTRIBUTOR LICENSE AGREEMENT 4 | 5 | Thank you for your interest in [Agoda.com](https://www.agoda.com) (the "Company") open source project. The Company asks you (“You” or “Contributor”) to sign this Contribution License Agreement in order to protect you as a Contributor and the Company and its users. This is not an assignment and it does not affect your ownership of your Contributions for any other purpose. It does, however, grant to the Company certain rights to your Contributions. 6 | 7 | Please complete, sign and return this Agreement to the Company below by using [GitHub](https://github.com) 8 | 9 | Please read this document carefully before signing and keep a copy for your records. 10 | 11 | - Full legal name: 12 | - Public name/Alias: 13 | - Mailing Address: 14 | - Location: 15 | - GitHub ID: 16 | - Telephone: 17 | - E-Mail: 18 | - Open Source Project(s):[Kafka JDBC Connector](https://github.com/agoda-com/kafka-jdbc-connector) 19 | 20 | You accept and agree to the following terms and conditions for Your past, present and future Contributions submitted to the Company. 21 | 22 | 1. Definitions. 23 | 24 | "You" (or "Your") shall mean the legal owner of the Contribution that is making this Agreement with the Company. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 25 | 26 | "Contribution" shall mean any original work of authorship, including any derivative works, modifications or additions to an existing work, that is intentionally submitted by You to the Company for inclusion in, or documentation of, any of the products owned or distributed by the Company (the "Work"). You acknowledge that the Company desires to have all contributions made by You under the terms of this CLA and, thus, this CLA will apply to all of your Contributions submitted both before and after the date of signature. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Company or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Company for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as: "Not a Contribution – Exempt from Contributor License Agreement dated _______.” 27 | 28 | 2. Grant of Copyright License. 29 | 30 | Subject to the terms and conditions of this Agreement, You hereby grant to the Company and to recipients of software distributed by the Company a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license, including a license, to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, distribute, or sell Your Contributions and the Work. 31 | 32 | 3. Grant of Patent License. 33 | 34 | Subject to the terms and conditions of this Agreement, You hereby grant to the Company and to recipients of the Work distributed by the Company a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable patent license to make, have made, use, offer to sell, sell, import, distribute, and otherwise transfer Your Contributions and the Work. 35 | 36 | 4. Ownership and third party rights. 37 | 38 | You represent that you are the sole legal owner and are legally entitled to grant the above license for the Contributions. If: 39 | 40 | - (A.) your employer(s) has intellectual property or other rights to your Contributions, you represent that you have both (i.) received express, prior, written permission to make Contributions on behalf of that employer; and (ii.) that your employer has waived any of its rights for or claims in your Contributions to the Company, or 41 | 42 | - (B.) if another individual or third party has rights to intellectual property to your Contributions – whether as a result of being a co-inventor, assignee, or other right, you represent that you have both (i.) received express, prior, written permission to make Contributions on behalf of that individual or third party; and (ii.) that such individual or third party has waived any of its rights for or claims in your Contributions to the Company. 43 | 44 | You will submit such written permission to the Company at the time of the submission of your Contribution. 45 | 46 | 5. Your original creation. 47 | 48 | You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include 49 | complete details (including required attributions and details of applicable license restrictions) of any third-party license or public domain licenses, or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware or should be aware and which are associated with any part of Your Contributions. 50 | 51 | 6. Support and warranties. 52 | 53 | You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, the Company acknowledges that You provide Your Contribution on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NONINFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. 54 | 55 | 7. Third party owned creation(s). 56 | 57 | Should You wish to submit work that is not Your original creation, You may submit it to the Company separately from any Contribution, clearly identifying the complete details of its ownership and source, and any applicable license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work, for example: "Submitted on behalf of a third-party: [named here].” “Owned by third-party: [named here.]” or “Copyright held by third-party: [named here].” 58 | 59 | 8. Notification. 60 | 61 | You agree to notify the Company of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect, including if you become aware of any third party intellectual property rights that are infringed by your Contributions. 62 | 63 | 9. Assignment. 64 | 65 | Neither party may assign this Agreement without the other party’s consent which will not be unreasonably withheld; however, each party may assign this Agreement without the other party’s consent to an entity or individual that acquires all or substantially all of the business or assets of the assigning party or for an individual acquires all of the intellectual property rights in the Contribution owned by such individual, whether by merger, sale of assets, or otherwise, provided that such entity or individual assumes and agrees in writing to be bound by all of the obligations of the assigning party under this Agreement. 66 | 67 | 10. Applicable law and jurisdiction. 68 | 69 | The Agreement shall be governed by and construed and enforced in accordance with, the laws of the State of New York, without reference to conflicts of laws. All disputes arising out of or relating to this Agreement will be submitted to the exclusive jurisdiction of the state or federal courts of the State of New York, and each party irrevocably consents to such personal jurisdiction and waives all objections to this venue. The application of the United Nations Convention on the International Sale of Goods to the Agreement is disclaimed in its entirety. 70 | 71 | 11. Entire agreement. 72 | 73 | This Agreement is the entire agreement, both written or oral, with respect to the Contributions between the parties. No amendment, modification or waiver of any provision of this Agreement will be effective unless in writing and signed by both parties. If any provision of this Agreement is held to be invalid or unenforceable, the remaining portions will remain in full force and effect and such provision will be enforced to the maximum extent possible so as to affect the intent of the parties and will be reformed to the extent necessary to make such provision valid and enforceable. All notices and other communications herein permitted or required under this Agreement will be sent by postage prepaid, via registered or certified mail or overnight courier, return receipt requested, or delivered personally to the parties at their respective addresses, or to such other address as either party will give to the other party in the manner provided herein for giving notice. Notice will be considered given upon receipt. 74 | 75 |

76 | 77 | \___________________________________ 78 | 79 | Contributor 80 | 81 | \___________________________________ 82 | 83 | Date 84 | 85 | \___________________________________ 86 | 87 | Print Name 88 | 89 |

90 | 91 | Acknowledged [Agoda.com](https://www.agoda.com) 92 | 93 | By: _________________________________ 94 | 95 | Date: _______________________________ 96 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. 6 | 7 | ## Our Standards 8 | 9 | Examples of behavior that contributes to creating a positive environment include: 10 | 11 | * Using welcoming and inclusive language 12 | * Being respectful of differing viewpoints and experiences 13 | * Gracefully accepting constructive criticism 14 | * Focusing on what is best for the community 15 | * Showing empathy towards other community members 16 | 17 | Examples of unacceptable behavior by participants include: 18 | 19 | * The use of sexualized language or imagery and unwelcome sexual attention or advances 20 | * Trolling, insulting/derogatory comments, and personal or political attacks 21 | * Public or private harassment 22 | * Publishing others' private information, such as a physical or electronic address, without explicit permission 23 | * Other conduct which could reasonably be considered inappropriate in a professional setting 24 | 25 | ## Our Responsibilities 26 | 27 | Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. 28 | 29 | Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. 30 | 31 | ## Scope 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. 34 | 35 | ## Enforcement 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at arpan.chaudhury@agoda.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. 38 | 39 | Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. 40 | 41 | ## Attribution 42 | 43 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] 44 | 45 | [homepage]: http://contributor-covenant.org 46 | [version]: http://contributor-covenant.org/version/1/4/ 47 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing to Kafka JDBC Connector 2 | ------------------------------------ 3 | 4 | In case you have questions about the contribution process or want to discuss about an issue please use the [kafka-connect-jdbc](https://gitter.im/kafka-jdbc-connector/Lobby) gitter channel. 5 | 6 | Tags 7 | ---- 8 | 9 | * Enhancement - This tag is assigned to an issue when it addresses an enhancement to the existing code-base. 10 | 11 | * Duplicate - This tag is assigned to an issue when there is already an open ticket for the same issue. 12 | 13 | * Bug - This tag is assigned to an issue when it addresses a bug in existing code-base. 14 | 15 | * Invalid - This tag is assigned to an issue when it does not fall under the scope of this project. 16 | 17 | General Workflow 18 | ---------------- 19 | 20 | The steps below describe how to get a patch into a master branch. 21 | 22 | 1. Find an open issue or create a new issue on [issue tracker](https://github.com/agoda-com/kafka-jdbc-connector/issues) for the work you want to contribute. Incase you wish to work on an existing issue make sure no one else is working on it. Comment on the issue declaring you are willing to take it up to avoid possible conflicts and reworks. Also avoid picking up issues tagged as Invalid. 23 | 2. [Fork the project](https://github.com/agoda-com/kafka-jdbc-connector#fork-destination-box) on GitHub. You'll need to create a feature-branch for your work on your fork, as this way you'll be able to submit a pull request against kafka-jdbc-connector. 24 | 3. Continue working on the feature branch until you are satisfied. Add tests for new features and modify existing tests if required. 25 | 4. Run all unit tests from sbt and make sure all of them pass. 26 | 5. Run code coverage to check if the lines of code you added are covered by unit tests. 27 | 6. Once your feature is complete, prepare the commit with appropriate message and the issue number. e.g. this commit is a sample #12. 28 | 7. Create a [pull request](https://help.github.com/articles/about-pull-requests/) and wait for the users to review. 29 | 8. Sign [Contributor License Agreement](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/CLA.md) if not already signed. 30 | 9. Once everything is said and done, your pull request gets merged. Your feature will be available with the next release. 31 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/agoda-com/kafka-jdbc-connector.svg?branch=master)](https://travis-ci.org/agoda-com/kafka-jdbc-connector) 2 | [![Gitter chat](https://badges.gitter.im/kafka-jdbc-connector/kafka-jdbc-connector.png)](https://gitter.im/kafka-jdbc-connector/Lobby) 3 | [![License: Apache](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/LICENSE.txt) 4 | [![Maven Central](https://maven-badges.herokuapp.com/maven-central/com.agoda/kafka-jdbc-connector_2.11/badge.svg)](https://maven-badges.herokuapp.com/maven-central/com.agoda/kafka-jdbc-connector_2.11) 5 | [![Codecov branch](https://img.shields.io/codecov/c/github/agoda-com/kafka-jdbc-connector/master.svg)](https://github.com/agoda-com/kafka-jdbc-connector) 6 | [![CLA assistant](https://cla-assistant.io/readme/badge/agoda-com/kafka-jdbc-connector)](https://cla-assistant.io/agoda-com/kafka-jdbc-connector) 7 | 8 | Kafka JDBC Connector 9 | ==================== 10 | 11 | *Simple way to copy data from relational databases into kafka.* 12 | 13 | To copy data between Kafka and another system, users create a Connector for the system which they want to pull data from or push data to. Connectors come in two flavors: *SourceConnectors* to import data from another system and *SinkConnectors* to export data from Kafka to other datasources. This project is an implementation of *SourceConnector* which allows users to copy data from relational databases into Kafka topics. It provides a flexible way to keep logic of selecting next batch of records inside database stored procedures which is invoked from the connector tasks in each poll cycle. 14 | 15 | Following are few advantages of using stored procedures 16 | 17 | * Select only required columns to be returned as a record. 18 | * Filtering / Grouping rows returned to connector with SQL. 19 | * Changing logic of returned batch without reconfiguring connector. 20 | 21 | Install 22 | ------- 23 | 24 | build.sbt 25 | 26 | ```scala 27 | libraryDependencies ++= Seq("com.agoda" %% "kafka-jdbc-connector" % "1.2.0") 28 | ``` 29 | 30 | Change Log 31 | ---------- 32 | 33 | Please refer to [Change Log](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/CHANGE_LOG.md) for requirements, supported databases and project changes. 34 | 35 | Examples 36 | -------- 37 | 38 | Click [here](https://github.com/agoda-com/kafka-jdbc-connector-samples) for examples. 39 | 40 | ### Timestamp mode 41 | 42 | Create a stored procedure in MSSQL database 43 | 44 | ``` 45 | create procedure [dbo].[cdc_table] 46 | @time datetime, 47 | @batch int 48 | as 49 | begin 50 | select top (@batch) * 51 | from cdc.table_ct as a 52 | left join cdc.lsn_time_mapping as b 53 | on a._$start_lsn = b.start_lsn 54 | where b.tran_end_time > @time 55 | order by b.tran_end_time asc 56 | end 57 | ``` 58 | 59 | Post the following configutation to Kafka Connect rest interface 60 | 61 | ``` 62 | { 63 | "name" : "cdc_timestamp", 64 | "config" : { 65 | "tasks.max": "1", 66 | "connector.class": "com.agoda.kafka.connector.jdbc.JdbcSourceConnector", 67 | "connection.url" : "jdbc:sqlserver://localhost:1433;user=sa;password=Passw0rd", 68 | "mode" : "timestamp", 69 | "stored-procedure.name" : "cdc_table", 70 | "topic" : "cdc-table-changelogs", 71 | "batch.max.rows.variable.name" : "batch", 72 | "timestamp.variable.name" : "time", 73 | "timestamp.field.name" : "tran_end_time" 74 | } 75 | } 76 | ``` 77 | 78 | ### Incrementing mode 79 | 80 | Create a stored procedure in MSSQL database 81 | 82 | ``` 83 | create procedure [dbo].[cdc_table] 84 | @id int, 85 | @batch int 86 | as 87 | begin 88 | select top (@batch) * 89 | from cdc.table_ct 90 | where auto_incrementing_id > @id 91 | order by auto_incrementing_id asc 92 | end 93 | ``` 94 | 95 | Post the following configutation to Kafka Connect rest interface 96 | 97 | ``` 98 | { 99 | "name" : "cdc_incrementing", 100 | "config" : { 101 | "tasks.max": "1", 102 | "connector.class": "com.agoda.kafka.connector.jdbc.JdbcSourceConnector", 103 | "connection.url" : "jdbc:sqlserver://localhost:1433;user=sa;password=Passw0rd", 104 | "mode" : "incrementing", 105 | "stored-procedure.name" : "cdc_table", 106 | "topic" : "cdc-table-changelogs", 107 | "batch.max.rows.variable.name" : "batch", 108 | "incrementing.variable.name" : "id", 109 | "incrementing.field.name" : "auto_incrementing_id" 110 | } 111 | } 112 | ``` 113 | 114 | ### Timestamp + Incrementing mode 115 | 116 | Create a stored procedure in MSSQL database 117 | 118 | ``` 119 | create procedure [dbo].[cdc_table] 120 | @time datetime, 121 | @id int, 122 | @batch int 123 | as 124 | begin 125 | select top (@batch) * 126 | from cdc.table_ct as a 127 | left join cdc.lsn_time_mapping as b 128 | on a._$start_lsn = b.start_lsn 129 | where b.tran_end_time > @time 130 | and a.auto_incrementing_id > @id 131 | order by b.tran_end_time, a.auto_incrementing_id asc 132 | end 133 | ``` 134 | 135 | Post the following configutation to Kafka Connect rest interface 136 | 137 | ``` 138 | { 139 | "name" : "cdc_timestamp_incrementing", 140 | "config" : { 141 | "tasks.max": "1", 142 | "connector.class": "com.agoda.kafka.connector.jdbc.JdbcSourceConnector", 143 | "connection.url" : "jdbc:sqlserver://localhost:1433;user=sa;password=Passw0rd", 144 | "mode" : "timestamp+incrementing", 145 | "stored-procedure.name" : "cdc_table", 146 | "topic" : "cdc-table-changelogs", 147 | "batch.max.rows.variable.name" : "batch", 148 | "timestamp.variable.name" : "time", 149 | "timestamp.field.name" : "tran_end_time", 150 | "incrementing.variable.name" : "id", 151 | "incrementing.field.name" : "auto_incrementing_id" 152 | } 153 | } 154 | ``` 155 | 156 | Contributing 157 | ------------ 158 | 159 | **Kafka JDBC Connector** is an open source project, and depends on its users to improve it. We are more than happy to find you interested in taking the project forward. 160 | 161 | Kindly refer to the [Contribution Guidelines](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/CONTRIBUTING.md) for detailed information. 162 | 163 | Code of Conduct 164 | --------------- 165 | 166 | Please refer to [Code of Conduct](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/CODE_OF_CONDUCT.md) document. 167 | 168 | License 169 | ------- 170 | 171 | Kafka JDBC Connector is Open Source and available under the [Apache License, Version 2.0](https://github.com/agoda-com/kafka-jdbc-connector/blob/master/LICENSE.txt). 172 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import sbt.Keys._ 3 | 4 | lazy val `kafka-jdbc-connector` = 5 | (project in file(".")) 6 | .settings( 7 | name := "kafka-jdbc-connector", 8 | version := "1.2.0", 9 | organization := "com.agoda", 10 | scalaVersion := "2.11.7", 11 | crossScalaVersions := Seq("2.11.7", "2.12.2"), 12 | libraryDependencies ++= Dependencies.Compile.kafkaJdbcConnector ++ Dependencies.Test.kafkaJdbcConnector, 13 | fork in Test := true 14 | ) 15 | .enablePlugins(BuildInfoPlugin) 16 | .settings( 17 | buildInfoKeys := Seq[BuildInfoKey](version), 18 | buildInfoPackage := organization.value 19 | ) 20 | .settings( 21 | test in assembly := {}, 22 | assemblyJarName in assembly := s"kafka-jdbc-connector-${version.value}.jar" 23 | ) 24 | .settings( 25 | useGpg := true, 26 | pgpPublicRing := file("~/.sbt/gpg/pubring.asc"), 27 | pgpSecretRing := file("~/.sbt/gpg/secring.asc"), 28 | publishTo := Some( 29 | if (isSnapshot.value) Opts.resolver.sonatypeSnapshots 30 | else Opts.resolver.sonatypeStaging 31 | ), 32 | credentials += Credentials(Path.userHome / ".ivy2" / ".credentials"), 33 | publishMavenStyle := true, 34 | licenses := Seq("Apache 2.0" -> url("https://www.apache.org/licenses/LICENSE-2.0")), 35 | homepage := Some(url("https://github.com/agoda-com/kafka-jdbc-connector")), 36 | scmInfo := Some( 37 | ScmInfo( 38 | url("https://github.com/agoda-com/kafka-jdbc-connector"), 39 | "scm:git@github.com:agoda-com/kafka-jdbc-connector.git" 40 | ) 41 | ), 42 | developers := List( 43 | Developer( 44 | id="arpanchaudhury", 45 | name="Arpan Chaudhury", 46 | email="arpan.chaudhury@agoda.com", 47 | url=url("https://github.com/arpanchaudhury") 48 | ) 49 | ) 50 | ) 51 | .settings( 52 | coverageExcludedPackages := Seq( 53 | "com.agoda.BuildInfo", 54 | "com.agoda.kafka.connector.jdbc.JdbcSourceConnector", 55 | "com.agoda.kafka.connector.jdbc.JdbcSourceTask", 56 | "com.agoda.kafka.connector.jdbc.utils.Version" 57 | ).mkString(";") 58 | ) 59 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnector.scala" 3 | - "src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceTask.scala" -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/agoda-com/kafka-jdbc-connector/280ae243ec2c6857f18f0ec2d8717d18b88914c7/logo.png -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | 3 | object Dependencies { 4 | private val ScalaTestV = "3.0.1" 5 | 6 | private val LogBack = "ch.qos.logback" % "logback-classic" % "1.2.3" 7 | private val ScalaLogging = "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0" 8 | private val KafkaConnectApi = "org.apache.kafka" % "connect-api" % "0.9.0.0" 9 | private val Enumeratum = "com.beachape" %% "enumeratum" % "1.5.12" 10 | private val Scalatics = "org.scalactic" %% "scalactic" % ScalaTestV % "test" 11 | private val ScalaTest = "org.scalatest" %% "scalatest" % ScalaTestV % "test" 12 | private val Mockito = "org.mockito" % "mockito-core" % "2.10.0" % "test" 13 | 14 | object Compile { 15 | def kafkaJdbcConnector = Seq(LogBack, ScalaLogging, KafkaConnectApi, Enumeratum) 16 | } 17 | 18 | object Test { 19 | def kafkaJdbcConnector = Seq(LogBack, ScalaLogging, Scalatics, ScalaTest, Mockito) 20 | } 21 | } -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.13 -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.6.1") 2 | 3 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.4") 4 | 5 | addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2") 6 | 7 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1") 8 | 9 | addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") 10 | 11 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.0") -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnector.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc 2 | 3 | import java.util 4 | 5 | import com.agoda.kafka.connector.jdbc.utils.Version 6 | import org.apache.kafka.connect.errors.ConnectException 7 | import org.apache.kafka.connect.source.{SourceConnector, SourceTask} 8 | import org.slf4j.LoggerFactory 9 | 10 | import scala.collection.JavaConverters._ 11 | import scala.util.{Failure, Success, Try} 12 | 13 | class JdbcSourceConnector extends SourceConnector { 14 | private val logger = LoggerFactory.getLogger(classOf[JdbcSourceConnector]) 15 | 16 | private var config: JdbcSourceConnectorConfig = _ 17 | 18 | /** 19 | * @return version of this connector 20 | */ 21 | override def version: String = Version.getVersion 22 | 23 | /** 24 | * invoked by kafka-connect runtime to start this connector 25 | * 26 | * @param props properties required to start this connector 27 | */ 28 | override def start(props: util.Map[String, String]): Unit = { 29 | Try (new JdbcSourceConnectorConfig(props.asScala.toMap)) match { 30 | case Success(c) => config = c 31 | case Failure(e) => logger.error("Couldn't start com.agoda.kafka.connector.jdbc.JdbcSourceConnector due to configuration error", new ConnectException(e)) 32 | } 33 | } 34 | 35 | /** 36 | * invoked by kafka-connect runtime to stop this connector 37 | */ 38 | override def stop(): Unit = { 39 | logger.debug("Stopping kafka source connector") 40 | } 41 | 42 | /** 43 | * invoked by kafka-connect runtime to instantiate SourceTask which polls data from external data store and saves into kafka 44 | * 45 | * @return class of source task to be created 46 | */ 47 | override def taskClass(): Class[_ <: SourceTask] = classOf[JdbcSourceTask] 48 | 49 | /** 50 | * returns a set of configurations for tasks based on the current configuration 51 | * 52 | * @param maxTasks maximum number of configurations to generate 53 | * @return configurations for tasks 54 | */ 55 | override def taskConfigs(maxTasks: Int): util.List[util.Map[String, String]] = List(config.properties.asJava).asJava 56 | } -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnectorConfig.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc 2 | 3 | import java.sql.Timestamp 4 | import java.util.TimeZone 5 | 6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants._ 7 | import com.agoda.kafka.connector.jdbc.models.Mode 8 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampIncrementingMode, TimestampMode} 9 | 10 | /** 11 | * @constructor 12 | * @param properties is set of configurations required to create JdbcSourceConnectorConfig 13 | */ 14 | class JdbcSourceConnectorConfig(val properties: Map[String, String]) { 15 | 16 | require( 17 | properties.contains(CONNECTION_URL_CONFIG) && 18 | properties.contains(MODE_CONFIG) && 19 | properties.contains(STORED_PROCEDURE_NAME_CONFIG) && 20 | properties.contains(TOPIC_CONFIG) && 21 | properties.contains(BATCH_MAX_ROWS_VARIABLE_NAME_CONFIG), 22 | s"""Required connector properties: 23 | | $CONNECTION_URL_CONFIG 24 | | $MODE_CONFIG 25 | | $STORED_PROCEDURE_NAME_CONFIG 26 | | $TOPIC_CONFIG 27 | | $BATCH_MAX_ROWS_VARIABLE_NAME_CONFIG""".stripMargin 28 | ) 29 | 30 | require( 31 | Mode.withName(properties(MODE_CONFIG)) match { 32 | case TimestampMode => properties.contains(TIMESTAMP_VARIABLE_NAME_CONFIG) && 33 | properties.contains(TIMESTAMP_FIELD_NAME_CONFIG) 34 | case IncrementingMode => properties.contains(INCREMENTING_VARIABLE_NAME_CONFIG) && 35 | properties.contains(INCREMENTING_FIELD_NAME_CONFIG) 36 | case TimestampIncrementingMode => properties.contains(TIMESTAMP_VARIABLE_NAME_CONFIG) && 37 | properties.contains(TIMESTAMP_FIELD_NAME_CONFIG) && 38 | properties.contains(INCREMENTING_VARIABLE_NAME_CONFIG) && 39 | properties.contains(INCREMENTING_FIELD_NAME_CONFIG) 40 | }, 41 | Mode.withName(properties(MODE_CONFIG)) match { 42 | case TimestampMode => s"""Required connector properties: 43 | | $TIMESTAMP_VARIABLE_NAME_CONFIG 44 | | $TIMESTAMP_FIELD_NAME_CONFIG""".stripMargin 45 | case IncrementingMode => s"""Required connector properties: 46 | | $INCREMENTING_VARIABLE_NAME_CONFIG 47 | | $INCREMENTING_FIELD_NAME_CONFIG""".stripMargin 48 | case TimestampIncrementingMode => s"""Required connector properties: 49 | | $TIMESTAMP_VARIABLE_NAME_CONFIG 50 | | $TIMESTAMP_FIELD_NAME_CONFIG 51 | | $INCREMENTING_VARIABLE_NAME_CONFIG 52 | | $INCREMENTING_FIELD_NAME_CONFIG""".stripMargin 53 | } 54 | ) 55 | 56 | /** 57 | * @return database connection url 58 | */ 59 | def getConnectionUrl: String = properties(CONNECTION_URL_CONFIG) 60 | 61 | /** 62 | * @return mode of operation [[IncrementingMode]], [[TimestampMode]], [[TimestampIncrementingMode]] 63 | */ 64 | def getMode: Mode = Mode.withName(properties(MODE_CONFIG)) 65 | 66 | /** 67 | * @return stored procedure name 68 | */ 69 | def getStoredProcedureName: String = properties(STORED_PROCEDURE_NAME_CONFIG) 70 | 71 | /** 72 | * @return kafka topic name 73 | */ 74 | def getTopic: String = properties(TOPIC_CONFIG) 75 | 76 | /** 77 | * @return database poll interval 78 | */ 79 | def getPollInterval: Long = properties.getOrElse(POLL_INTERVAL_MS_CONFIG, POLL_INTERVAL_MS_DEFAULT).toLong 80 | 81 | /** 82 | * @return number of records fetched in each poll 83 | */ 84 | def getMaxBatchSize: Int = properties.getOrElse(BATCH_MAX_ROWS_CONFIG, BATCH_MAX_ROWS_DEFAULT).toInt 85 | 86 | /** 87 | * @return batch size variable name in stored procedure 88 | */ 89 | def getMaxBatchSizeVariableName: String = properties(BATCH_MAX_ROWS_VARIABLE_NAME_CONFIG) 90 | 91 | /** 92 | * @return timestamp offset variable name in stored procedure 93 | */ 94 | def getTimestampVariableName: Option[String] = properties.get(TIMESTAMP_VARIABLE_NAME_CONFIG) 95 | 96 | /** 97 | * @return timestamp offset field name in record 98 | */ 99 | def getTimestampFieldName: Option[String] = properties.get(TIMESTAMP_FIELD_NAME_CONFIG) 100 | 101 | /** 102 | * @return incrementing offset variable name in stored procedure 103 | */ 104 | def getIncrementingVariableName: Option[String] = properties.get(INCREMENTING_VARIABLE_NAME_CONFIG) 105 | 106 | /** 107 | * @return incrementing offset field name in record 108 | */ 109 | def getIncrementingFieldName: Option[String] = properties.get(INCREMENTING_FIELD_NAME_CONFIG) 110 | 111 | /** 112 | * @return initial timestamp offset 113 | */ 114 | def getTimestampOffset: Long = { 115 | properties 116 | .get(TIMESTAMP_OFFSET_CONFIG) 117 | .map(o => new Timestamp(Timestamp.valueOf(o).getTime + TimeZone.getDefault.getRawOffset)) 118 | .getOrElse(TIMESTAMP_OFFSET_DEFAULT).getTime 119 | } 120 | 121 | /** 122 | * @return initial incrementing offset 123 | */ 124 | def getIncrementingOffset: Long = properties.getOrElse(INCREMENTING_OFFSET_CONFIG, INCREMENTING_OFFSET_DEFAULT).toLong 125 | 126 | /** 127 | * @return optional field name to be used as kafka message key 128 | */ 129 | def getKeyField: Option[String] = properties.get(KEY_FIELD_NAME_CONFIG) 130 | } -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnectorConstants.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc 2 | 3 | import java.sql.Timestamp 4 | 5 | object JdbcSourceConnectorConstants { 6 | 7 | val STORED_PROCEDURE_NAME_KEY = "stored-procedure.name" 8 | 9 | val CONNECTION_URL_CONFIG = "connection.url" 10 | 11 | val MODE_CONFIG = "mode" 12 | val TIMESTAMP_VARIABLE_NAME_CONFIG = "timestamp.variable.name" 13 | val TIMESTAMP_FIELD_NAME_CONFIG = "timestamp.field.name" 14 | val INCREMENTING_VARIABLE_NAME_CONFIG = "incrementing.variable.name" 15 | val INCREMENTING_FIELD_NAME_CONFIG = "incrementing.field.name" 16 | 17 | val STORED_PROCEDURE_NAME_CONFIG = "stored-procedure.name" 18 | 19 | val TOPIC_CONFIG = "topic" 20 | 21 | val POLL_INTERVAL_MS_CONFIG = "poll.interval.ms" 22 | val POLL_INTERVAL_MS_DEFAULT = "5000" 23 | 24 | val BATCH_MAX_ROWS_VARIABLE_NAME_CONFIG = "batch.max.rows.variable.name" 25 | val BATCH_MAX_ROWS_CONFIG = "batch.max.records" 26 | val BATCH_MAX_ROWS_DEFAULT = "100" 27 | 28 | val TIMESTAMP_OFFSET_CONFIG = "timestamp.offset" 29 | val TIMESTAMP_OFFSET_DEFAULT = new Timestamp(0L) 30 | val INCREMENTING_OFFSET_CONFIG = "incrementing.offset" 31 | val INCREMENTING_OFFSET_DEFAULT = "0" 32 | 33 | val KEY_FIELD_NAME_CONFIG = "key.field.name" 34 | 35 | val TASKS_MAX_CONFIG = "tasks.max" 36 | val CONNECTOR_CLASS = "connector.class" 37 | } 38 | -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceTask.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc 2 | 3 | import java.sql.{Connection, DriverManager, SQLException} 4 | import java.util 5 | import java.util.concurrent.atomic.AtomicBoolean 6 | 7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct 8 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampIncrementingMode, TimestampMode} 9 | import com.agoda.kafka.connector.jdbc.services.{DataService, IdBasedDataService, TimeBasedDataService, TimeIdBasedDataService} 10 | import com.agoda.kafka.connector.jdbc.utils.{DataConverter, Version} 11 | import org.apache.kafka.connect.errors.ConnectException 12 | import org.apache.kafka.connect.source.{SourceRecord, SourceTask} 13 | import org.slf4j.LoggerFactory 14 | 15 | import scala.collection.JavaConverters._ 16 | import scala.concurrent.duration._ 17 | import scala.util.{Failure, Success, Try} 18 | 19 | class JdbcSourceTask extends SourceTask { 20 | private val logger = LoggerFactory.getLogger(classOf[JdbcSourceTask]) 21 | private val dataConverter = new DataConverter 22 | 23 | private var config: JdbcSourceTaskConfig = _ 24 | private var db: Connection = _ 25 | private var dataService: DataService = _ 26 | private var running: AtomicBoolean = _ 27 | 28 | override def version(): String = Version.getVersion 29 | 30 | /** 31 | * invoked by kafka-connect runtime to start this task 32 | * 33 | * @param props properties required to start this task 34 | */ 35 | override def start(props: util.Map[String, String]): Unit = { 36 | Try(new JdbcSourceTaskConfig(props.asScala.toMap)) match { 37 | case Success(c) => config = c 38 | case Failure(e) => logger.error("Couldn't start com.agoda.kafka.connector.jdbc.JdbcSourceTask due to configuration error", new ConnectException(e)) 39 | } 40 | 41 | val dbUrl = config.getConnectionUrl 42 | logger.debug(s"Trying to connect to $dbUrl") 43 | Try(DriverManager.getConnection(dbUrl)) match { 44 | case Success(c) => db = c 45 | case Failure(e: SQLException) => logger.error(s"Couldn't open connection to $dbUrl : ", e) 46 | throw new ConnectException(e) 47 | case Failure(e) => logger.error(s"Couldn't open connection to $dbUrl : ", e) 48 | throw e 49 | } 50 | 51 | val databaseProduct = DatabaseProduct.withName(db.getMetaData.getDatabaseProductName) 52 | 53 | val offset = context.offsetStorageReader().offset( 54 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> config.getStoredProcedureName).asJava 55 | ) 56 | 57 | val storedProcedureName = config.getStoredProcedureName 58 | val timestampVariableNameOpt = config.getTimestampVariableName 59 | val timestampFieldNameOpt = config.getTimestampFieldName 60 | val incrementingVariableNameOpt = config.getIncrementingVariableName 61 | val incrementingFieldNameOpt = config.getIncrementingFieldName 62 | val batchSize = config.getMaxBatchSize 63 | val batchSizeVariableName = config.getMaxBatchSizeVariableName 64 | val topic = config.getTopic 65 | val keyFieldOpt = config.getKeyField 66 | 67 | config.getMode match { 68 | case TimestampMode => 69 | val timestampOffset = Try(offset.get(TimestampMode.entryName)).map(_.toString.toLong).getOrElse(config.getTimestampOffset) 70 | dataService = TimeBasedDataService(databaseProduct, storedProcedureName, batchSize, batchSizeVariableName, 71 | timestampVariableNameOpt.get, timestampOffset, timestampFieldNameOpt.get, topic, keyFieldOpt, dataConverter) 72 | 73 | case IncrementingMode => 74 | val incrementingOffset = Try(offset.get(IncrementingMode.entryName)).map(_.toString.toLong).getOrElse(config.getIncrementingOffset) 75 | dataService = IdBasedDataService(databaseProduct, storedProcedureName, batchSize, batchSizeVariableName, 76 | incrementingVariableNameOpt.get, incrementingOffset, incrementingFieldNameOpt.get, topic, keyFieldOpt, dataConverter) 77 | 78 | case TimestampIncrementingMode => 79 | val timestampOffset = Try(offset.get(TimestampMode.entryName)).map(_.toString.toLong).getOrElse(config.getTimestampOffset) 80 | val incrementingOffset = Try(offset.get(IncrementingMode.entryName)).map(_.toString.toLong).getOrElse(config.getIncrementingOffset) 81 | dataService = TimeIdBasedDataService(databaseProduct, storedProcedureName, batchSize, batchSizeVariableName, 82 | timestampVariableNameOpt.get, timestampOffset, incrementingVariableNameOpt.get, incrementingOffset, 83 | timestampFieldNameOpt.get, incrementingFieldNameOpt.get, topic, keyFieldOpt, dataConverter) 84 | } 85 | 86 | running = new AtomicBoolean(true) 87 | } 88 | 89 | /** 90 | * invoked by kafka-connect runtime to stop this task 91 | */ 92 | override def stop(): Unit = { 93 | if (running != null) running.set(false) 94 | if (db != null) { 95 | logger.debug("Trying to close database connection") 96 | Try(db.close()) match { 97 | case Success(_) => 98 | case Failure(e) => logger.error("Failed to close database connection: ", e) 99 | } 100 | } 101 | } 102 | 103 | /** 104 | * invoked by kafka-connect runtime to poll data in [[JdbcSourceConnectorConstants.POLL_INTERVAL_MS_CONFIG]] interval 105 | */ 106 | override def poll(): util.List[SourceRecord] = this.synchronized { if(running.get) fetchRecords else null } 107 | 108 | private def fetchRecords: util.List[SourceRecord] = { 109 | logger.debug("Polling new data ...") 110 | val pollInterval = config.getPollInterval 111 | val startTime = System.currentTimeMillis 112 | val fetchedRecords = dataService.getRecords(db, pollInterval.millis) match { 113 | case Success(records) => if(records.isEmpty) logger.info(s"No updates for $dataService") 114 | else logger.info(s"Returning ${records.size} records for $dataService") 115 | records 116 | case Failure(e: SQLException) => logger.error(s"Failed to fetch data for $dataService: ", e) 117 | Seq.empty[SourceRecord] 118 | case Failure(e: Throwable) => logger.error(s"Failed to fetch data for $dataService: ", e) 119 | Seq.empty[SourceRecord] 120 | } 121 | val endTime = System.currentTimeMillis 122 | val elapsedTime = endTime - startTime 123 | 124 | if(elapsedTime < pollInterval) Thread.sleep(pollInterval - elapsedTime) 125 | fetchedRecords.asJava 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/JdbcSourceTaskConfig.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc 2 | 3 | /** 4 | * @constructor 5 | * @param properties is set of configurations required to create JdbcSourceTaskConfig 6 | */ 7 | class JdbcSourceTaskConfig(properties: Map[String, String]) extends JdbcSourceConnectorConfig(properties) -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/models/DatabaseProduct.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.models 2 | 3 | import enumeratum._ 4 | 5 | import scala.collection.immutable.IndexedSeq 6 | 7 | /** 8 | * Database Product Name 9 | * ~~~~~~~~~~~~~~~~~~~~~ 10 | * 11 | * MsSQL :: Microsoft SQL Server. 12 | * 13 | * MySQL :: MySQL Server. 14 | * 15 | */ 16 | sealed abstract class DatabaseProduct(override val entryName: String) extends EnumEntry 17 | 18 | object DatabaseProduct extends Enum[DatabaseProduct] { 19 | 20 | val values: IndexedSeq[DatabaseProduct] = findValues 21 | 22 | case object MsSQL extends DatabaseProduct("Microsoft SQL Server") 23 | case object MySQL extends DatabaseProduct("MySQL") 24 | } -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/models/Mode.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.models 2 | 3 | import enumeratum._ 4 | 5 | import scala.collection.immutable.IndexedSeq 6 | 7 | /** 8 | * Mode of operation 9 | * ~~~~~~~~~~~~~~~~~ 10 | * 11 | * Timestamp Mode :: creation timestamp of a record is stored as offset. 12 | * 13 | * Incrementing Mode :: unique (auto) incrementing integral id of a record is stored as offset. 14 | * 15 | * Timestamp + Incrementing Mode :: pair of creation timestamp and unique (auto) incrementing integral id of a record 16 | * is stored as offset. 17 | */ 18 | sealed abstract class Mode(override val entryName: String) extends EnumEntry 19 | 20 | object Mode extends Enum[Mode] { 21 | 22 | val values: IndexedSeq[Mode] = findValues 23 | 24 | case object TimestampMode extends Mode("timestamp") 25 | case object IncrementingMode extends Mode("incrementing") 26 | case object TimestampIncrementingMode extends Mode("timestamp+incrementing") 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/services/DataService.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.services 2 | 3 | import java.sql.{Connection, PreparedStatement, ResultSet} 4 | 5 | import com.agoda.kafka.connector.jdbc.utils.DataConverter 6 | import org.apache.kafka.connect.data.Schema 7 | import org.apache.kafka.connect.source.SourceRecord 8 | 9 | import scala.concurrent.duration.Duration 10 | import scala.util.Try 11 | 12 | trait DataService { 13 | 14 | /** 15 | * @return name of the stored procedure 16 | */ 17 | def storedProcedureName: String 18 | 19 | /** 20 | * Utility to convert SQL ResultSet into Struct 21 | * 22 | * @return instance of DataConverter 23 | */ 24 | def dataConverter: DataConverter 25 | 26 | /** 27 | * Fetch records from database 28 | * 29 | * @param connection database connection 30 | * @param timeout query timeout 31 | * @return Success(Seq(SourceRecord)) if records are processed successfully else Failure(Throwable) 32 | */ 33 | def getRecords(connection: Connection, timeout: Duration): Try[Seq[SourceRecord]] = { 34 | for { 35 | preparedStatement <- createPreparedStatement(connection) 36 | resultSet <- executeStoredProcedure(preparedStatement, timeout) 37 | schema <- dataConverter.convertSchema(storedProcedureName, resultSet.getMetaData) 38 | records <- extractRecords(resultSet, schema) 39 | } yield records 40 | } 41 | 42 | protected def createPreparedStatement(connection: Connection): Try[PreparedStatement] 43 | 44 | protected def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]] 45 | 46 | private def executeStoredProcedure(preparedStatement: PreparedStatement, timeout: Duration): Try[ResultSet] = Try { 47 | preparedStatement.setQueryTimeout(timeout.toSeconds.toInt) 48 | preparedStatement.executeQuery 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/services/IdBasedDataService.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.services 2 | 3 | import java.io.IOException 4 | import java.sql.{Connection, PreparedStatement, ResultSet} 5 | 6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants 7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct 8 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL} 9 | import com.agoda.kafka.connector.jdbc.models.Mode.IncrementingMode 10 | import com.agoda.kafka.connector.jdbc.utils.DataConverter 11 | import org.apache.kafka.connect.data.Schema 12 | import org.apache.kafka.connect.data.Schema.Type 13 | import org.apache.kafka.connect.source.SourceRecord 14 | import org.slf4j.LoggerFactory 15 | 16 | import scala.collection.JavaConverters._ 17 | import scala.collection.mutable.ListBuffer 18 | import scala.util.Try 19 | 20 | /** 21 | * @constructor 22 | * @param databaseProduct type of database server 23 | * @param storedProcedureName name of the stored procedure 24 | * @param batchSize number of records returned in each batch 25 | * @param batchSizeVariableName name of the batch size variable in stored procedure 26 | * @param incrementingVariableName name of the incrementing offset variable in stored procedure 27 | * @param incrementingOffset value of current incrementing offset 28 | * @param incrementingFieldName incrementing offset field name in returned records 29 | * @param topic name of kafka topic where records are stored 30 | * @param keyFieldOpt optional key field name in returned records 31 | * @param dataConverter ResultSet converter utility 32 | */ 33 | case class IdBasedDataService(databaseProduct: DatabaseProduct, 34 | storedProcedureName: String, 35 | batchSize: Int, 36 | batchSizeVariableName: String, 37 | incrementingVariableName: String, 38 | var incrementingOffset: Long, 39 | incrementingFieldName: String, 40 | topic: String, 41 | keyFieldOpt: Option[String], 42 | dataConverter: DataConverter) extends DataService { 43 | private val logger = LoggerFactory.getLogger(this.getClass) 44 | 45 | override def createPreparedStatement(connection: Connection): Try[PreparedStatement] = Try { 46 | val preparedStatement = databaseProduct match { 47 | case MsSQL => connection.prepareStatement(s"EXECUTE $storedProcedureName @$incrementingVariableName = ?, @$batchSizeVariableName = ?") 48 | case MySQL => connection.prepareStatement(s"CALL $storedProcedureName (@$incrementingVariableName := ?, @$batchSizeVariableName := ?)") 49 | } 50 | preparedStatement.setObject(1, incrementingOffset) 51 | preparedStatement.setObject(2, batchSize) 52 | preparedStatement 53 | } 54 | 55 | override def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]] = Try { 56 | val sourceRecords = ListBuffer.empty[SourceRecord] 57 | val idSchemaType = schema.field(incrementingFieldName).schema.`type`() 58 | var max = incrementingOffset 59 | while(resultSet.next()) { 60 | dataConverter.convertRecord(schema, resultSet).map { record => 61 | val id = idSchemaType match { 62 | case Type.INT8 => record.getInt8(incrementingFieldName).toLong 63 | case Type.INT16 => record.getInt16(incrementingFieldName).toLong 64 | case Type.INT32 => record.getInt32(incrementingFieldName).toLong 65 | case Type.INT64 => record.getInt64(incrementingFieldName).toLong 66 | case _ => 67 | logger.warn("Id field is not of type INT") 68 | throw new IOException("Id field is not of type INT") 69 | } 70 | max = if (id > max) { 71 | keyFieldOpt match { 72 | case Some(keyField) => 73 | sourceRecords += new SourceRecord( 74 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava, 75 | Map(IncrementingMode.entryName -> id).asJava, topic, null, schema, record.get(keyField), schema, record 76 | ) 77 | case None => 78 | sourceRecords += new SourceRecord( 79 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava, 80 | Map(IncrementingMode.entryName -> id).asJava, topic, schema, record 81 | ) 82 | } 83 | id 84 | } else max 85 | } 86 | } 87 | incrementingOffset = max 88 | sourceRecords 89 | } 90 | 91 | override def toString: String = { 92 | s""" 93 | |{ 94 | | "name" : "${this.getClass.getSimpleName}" 95 | | "mode" : "${IncrementingMode.entryName}" 96 | | "stored-procedure.name" : "$storedProcedureName" 97 | |} 98 | """.stripMargin 99 | } 100 | } -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/services/TimeBasedDataService.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.services 2 | 3 | import java.sql.{Connection, PreparedStatement, ResultSet, Timestamp} 4 | import java.util.{Date, GregorianCalendar, TimeZone} 5 | 6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants 7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct 8 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL} 9 | import com.agoda.kafka.connector.jdbc.models.Mode.TimestampMode 10 | import com.agoda.kafka.connector.jdbc.utils.DataConverter 11 | import org.apache.kafka.connect.data.Schema 12 | import org.apache.kafka.connect.source.SourceRecord 13 | 14 | import scala.collection.JavaConverters._ 15 | import scala.collection.mutable.ListBuffer 16 | import scala.util.Try 17 | 18 | /** 19 | * @constructor 20 | * @param databaseProduct type of database server 21 | * @param storedProcedureName name of the stored procedure 22 | * @param batchSize number of records returned in each batch 23 | * @param batchSizeVariableName name of the batch size variable in stored procedure 24 | * @param timestampVariableName name of the timestamp offset variable in stored procedure 25 | * @param timestampOffset value of current timestamp offset 26 | * @param timestampFieldName timestamp offset field name in returned records 27 | * @param topic name of kafka topic where records are stored 28 | * @param keyFieldOpt optional key field name in returned records 29 | * @param dataConverter ResultSet converter utility 30 | */ 31 | case class TimeBasedDataService(databaseProduct: DatabaseProduct, 32 | storedProcedureName: String, 33 | batchSize: Int, 34 | batchSizeVariableName: String, 35 | timestampVariableName: String, 36 | var timestampOffset: Long, 37 | timestampFieldName: String, 38 | topic: String, 39 | keyFieldOpt: Option[String], 40 | dataConverter: DataConverter, 41 | calendar: GregorianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")) 42 | ) extends DataService { 43 | 44 | override def createPreparedStatement(connection: Connection): Try[PreparedStatement] = Try { 45 | val preparedStatement = databaseProduct match { 46 | case MsSQL => connection.prepareStatement(s"EXECUTE $storedProcedureName @$timestampVariableName = ?, @$batchSizeVariableName = ?") 47 | case MySQL => connection.prepareStatement(s"CALL $storedProcedureName (@$timestampVariableName := ?, @$batchSizeVariableName := ?)") 48 | } 49 | preparedStatement.setTimestamp(1, new Timestamp(timestampOffset), calendar) 50 | preparedStatement.setObject(2, batchSize) 51 | preparedStatement 52 | } 53 | 54 | override def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]] = Try { 55 | val sourceRecords = ListBuffer.empty[SourceRecord] 56 | var max = timestampOffset 57 | while (resultSet.next()) { 58 | dataConverter.convertRecord(schema, resultSet) map { record => 59 | val time = record.get(timestampFieldName).asInstanceOf[Date].getTime 60 | max = if(time > max) { 61 | keyFieldOpt match { 62 | case Some(keyField) => 63 | sourceRecords += new SourceRecord( 64 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava, 65 | Map(TimestampMode.entryName -> time).asJava, topic, null, schema, record.get(keyField), schema, record 66 | ) 67 | case None => 68 | sourceRecords += new SourceRecord( 69 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava, 70 | Map(TimestampMode.entryName -> time).asJava, topic, schema, record 71 | ) 72 | } 73 | time 74 | } else max 75 | } 76 | } 77 | timestampOffset = max 78 | sourceRecords 79 | } 80 | 81 | override def toString: String = { 82 | s""" 83 | |{ 84 | | "name" : "${this.getClass.getSimpleName}" 85 | | "mode" : "${TimestampMode.entryName}" 86 | | "stored-procedure.name" : "$storedProcedureName" 87 | |} 88 | """.stripMargin 89 | } 90 | } -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/services/TimeIdBasedDataService.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.services 2 | 3 | import java.io.IOException 4 | import java.sql.{Connection, PreparedStatement, ResultSet, Timestamp} 5 | import java.util.{Date, GregorianCalendar, TimeZone} 6 | 7 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants 8 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct 9 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL} 10 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampMode} 11 | import com.agoda.kafka.connector.jdbc.utils.DataConverter 12 | import org.apache.kafka.connect.data.Schema 13 | import org.apache.kafka.connect.data.Schema.Type 14 | import org.apache.kafka.connect.source.SourceRecord 15 | import org.slf4j.LoggerFactory 16 | 17 | import scala.collection.JavaConverters._ 18 | import scala.collection.mutable.ListBuffer 19 | import scala.util.Try 20 | 21 | /** 22 | * @constructor 23 | * @param databaseProduct type of database server 24 | * @param storedProcedureName name of the stored procedure 25 | * @param batchSize number of records returned in each batch 26 | * @param batchSizeVariableName name of the batch size variable in stored procedure 27 | * @param timestampVariableName name of the timestamp offset variable in stored procedure 28 | * @param timestampOffset value of current timestamp offset 29 | * @param timestampFieldName timestamp offset field name in returned records 30 | * @param incrementingVariableName name of the incrementing offset variable in stored procedure 31 | * @param incrementingOffset value of current incrementing offset 32 | * @param incrementingFieldName incrementing offset field name in returned records 33 | * @param topic name of kafka topic where records are stored 34 | * @param keyFieldOpt optional key field name in returned records 35 | * @param dataConverter ResultSet converter utility 36 | */ 37 | case class TimeIdBasedDataService(databaseProduct: DatabaseProduct, 38 | storedProcedureName: String, 39 | batchSize: Int, 40 | batchSizeVariableName: String, 41 | timestampVariableName: String, 42 | var timestampOffset: Long, 43 | incrementingVariableName: String, 44 | var incrementingOffset: Long, 45 | timestampFieldName: String, 46 | incrementingFieldName: String, 47 | topic: String, 48 | keyFieldOpt: Option[String], 49 | dataConverter: DataConverter, 50 | calendar: GregorianCalendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")) 51 | ) extends DataService { 52 | private val logger = LoggerFactory.getLogger(this.getClass) 53 | 54 | override def createPreparedStatement(connection: Connection): Try[PreparedStatement] = Try { 55 | val preparedStatement = databaseProduct match { 56 | case MsSQL => connection.prepareStatement(s"EXECUTE $storedProcedureName @$timestampVariableName = ?, @$incrementingVariableName = ?, @$batchSizeVariableName = ?") 57 | case MySQL => connection.prepareStatement(s"CALL $storedProcedureName (@$timestampVariableName := ?, @$incrementingVariableName := ?, @$batchSizeVariableName := ?)") 58 | } 59 | preparedStatement.setTimestamp(1, new Timestamp(timestampOffset), calendar) 60 | preparedStatement.setObject(2, incrementingOffset) 61 | preparedStatement.setObject(3, batchSize) 62 | preparedStatement 63 | } 64 | 65 | override def extractRecords(resultSet: ResultSet, schema: Schema): Try[Seq[SourceRecord]] = Try { 66 | val sourceRecords = ListBuffer.empty[SourceRecord] 67 | var maxTime = timestampOffset 68 | var maxId = incrementingOffset 69 | val idSchemaType = schema.field(incrementingFieldName).schema.`type`() 70 | while (resultSet.next()) { 71 | dataConverter.convertRecord(schema, resultSet) map { record => 72 | val time = record.get(timestampFieldName).asInstanceOf[Date].getTime 73 | val id = idSchemaType match { 74 | case Type.INT8 => record.getInt8(incrementingFieldName).toLong 75 | case Type.INT16 => record.getInt16(incrementingFieldName).toLong 76 | case Type.INT32 => record.getInt32(incrementingFieldName).toLong 77 | case Type.INT64 => record.getInt64(incrementingFieldName).toLong 78 | case _ => 79 | logger.warn("Id field is not of type INT") 80 | throw new IOException("Id field is not of type INT") 81 | } 82 | 83 | maxTime = if(time > maxTime) time else maxTime 84 | maxId = if (id > maxId) id else maxId 85 | 86 | if(time >= timestampOffset && id > incrementingOffset) { 87 | keyFieldOpt match { 88 | case Some(keyField) => 89 | sourceRecords += new SourceRecord( 90 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava, 91 | Map(TimestampMode.entryName -> time, IncrementingMode.entryName -> id).asJava, 92 | topic, null, schema, record.get(keyField), schema, record 93 | ) 94 | case None => 95 | sourceRecords += new SourceRecord( 96 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> storedProcedureName).asJava, 97 | Map(TimestampMode.entryName -> time, IncrementingMode.entryName -> id).asJava, 98 | topic, schema, record 99 | ) 100 | } 101 | } 102 | } 103 | } 104 | timestampOffset = maxTime 105 | incrementingOffset = maxId 106 | sourceRecords 107 | } 108 | 109 | override def toString: String = { 110 | s""" 111 | |{ 112 | | "name" : "${this.getClass.getSimpleName}" 113 | | "mode" : "${TimestampMode.entryName}+${IncrementingMode.entryName}" 114 | | "stored-procedure.name" : "$storedProcedureName" 115 | |} 116 | """.stripMargin 117 | } 118 | } -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/utils/DataConverter.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.utils 2 | 3 | import java.io.IOException 4 | import java.sql.{ResultSet, ResultSetMetaData, Types} 5 | import java.util.{GregorianCalendar, TimeZone} 6 | 7 | import com.agoda.kafka.connector.jdbc.JdbcSourceTask 8 | import org.apache.kafka.connect.data._ 9 | import org.slf4j.LoggerFactory 10 | 11 | import scala.util.Try 12 | 13 | class DataConverter { 14 | private val logger = LoggerFactory.getLogger(this.getClass) 15 | private val UTC_CALENDAR = new GregorianCalendar(TimeZone.getTimeZone("UTC")) 16 | 17 | /** 18 | * Create schema from result set returned by stored procedure 19 | * 20 | * @param storedProcedureName name of the stored procedure, used as schema name 21 | * @param metadata metadata of result set returned by the stored procedure 22 | * @return Success(Schema) if schema is created successfully else Failure(Throwable) 23 | */ 24 | def convertSchema(storedProcedureName: String, metadata: ResultSetMetaData): Try[Schema] = Try { 25 | val builder = SchemaBuilder.struct.name(storedProcedureName) 26 | (1 to metadata.getColumnCount).foreach(i => addFieldSchema(metadata, i, builder)) 27 | builder.build 28 | } 29 | 30 | /** 31 | * Convert result set row into structured object 32 | * 33 | * @param schema schema created from result set metadata 34 | * @param resultSet result set returned by the stored procedure 35 | * @return Success(Struct) if structured object is created successfully else Failure(Throwable) 36 | */ 37 | def convertRecord(schema: Schema, resultSet: ResultSet): Try[Struct] = Try { 38 | val metadata = resultSet.getMetaData 39 | val struct = new Struct(schema) 40 | (1 to metadata.getColumnCount).foreach { i => 41 | convertFieldValue(resultSet, i, metadata.getColumnType(i), struct, metadata.getColumnLabel(i)) 42 | } 43 | struct 44 | } 45 | 46 | private def addFieldSchema(metadata: ResultSetMetaData, col: Int, builder: SchemaBuilder) = { 47 | val label = metadata.getColumnLabel(col) 48 | val name = metadata.getColumnName(col) 49 | val fieldName = if (label != null && !label.isEmpty) label else name 50 | val sqlType = metadata.getColumnType(col) 51 | val optional = metadata.isNullable(col) == ResultSetMetaData.columnNullable || 52 | metadata.isNullable(col) == ResultSetMetaData.columnNullableUnknown 53 | 54 | sqlType match { 55 | case Types.BOOLEAN => 56 | if (optional) builder.field(fieldName, Schema.OPTIONAL_BOOLEAN_SCHEMA) 57 | else builder.field(fieldName, Schema.BOOLEAN_SCHEMA) 58 | 59 | case Types.BIT | Types.TINYINT => 60 | if (optional) builder.field(fieldName, Schema.OPTIONAL_INT8_SCHEMA) 61 | else builder.field(fieldName, Schema.INT8_SCHEMA) 62 | 63 | case Types.SMALLINT => 64 | if (optional) builder.field(fieldName, Schema.OPTIONAL_INT16_SCHEMA) 65 | else builder.field(fieldName, Schema.INT16_SCHEMA) 66 | 67 | case Types.INTEGER => 68 | if (optional) builder.field(fieldName, Schema.OPTIONAL_INT32_SCHEMA) 69 | else builder.field(fieldName, Schema.INT32_SCHEMA) 70 | 71 | case Types.BIGINT => 72 | if (optional) builder.field(fieldName, Schema.OPTIONAL_INT64_SCHEMA) 73 | else builder.field(fieldName, Schema.INT64_SCHEMA) 74 | 75 | case Types.REAL => 76 | if (optional) builder.field(fieldName, Schema.OPTIONAL_FLOAT32_SCHEMA) 77 | else builder.field(fieldName, Schema.FLOAT32_SCHEMA) 78 | 79 | case Types.FLOAT | Types.DOUBLE => 80 | if (optional) builder.field(fieldName, Schema.OPTIONAL_FLOAT64_SCHEMA) 81 | else builder.field(fieldName, Schema.FLOAT64_SCHEMA) 82 | 83 | case Types.NUMERIC | Types.DECIMAL => 84 | val fieldBuilder = Decimal.builder(metadata.getScale(col)) 85 | if (optional) fieldBuilder.optional 86 | builder.field(fieldName, fieldBuilder.build) 87 | 88 | case Types.CHAR | Types.VARCHAR | Types.LONGVARCHAR | Types.NCHAR | Types.NVARCHAR | 89 | Types.LONGNVARCHAR | Types.CLOB | Types.NCLOB | Types.DATALINK | Types.SQLXML => 90 | if (optional) builder.field(fieldName, Schema.OPTIONAL_STRING_SCHEMA) 91 | else builder.field(fieldName, Schema.STRING_SCHEMA) 92 | 93 | case Types.BINARY | Types.BLOB | Types.VARBINARY | Types.LONGVARBINARY => 94 | if (optional) builder.field(fieldName, Schema.OPTIONAL_BYTES_SCHEMA) 95 | else builder.field(fieldName, Schema.BYTES_SCHEMA) 96 | 97 | case Types.DATE => 98 | val dateSchemaBuilder = Date.builder 99 | if (optional) dateSchemaBuilder.optional 100 | builder.field(fieldName, dateSchemaBuilder.build) 101 | 102 | case Types.TIME => 103 | val timeSchemaBuilder = Time.builder 104 | if (optional) timeSchemaBuilder.optional 105 | builder.field(fieldName, timeSchemaBuilder.build) 106 | 107 | case Types.TIMESTAMP => 108 | val tsSchemaBuilder = Timestamp.builder 109 | if (optional) tsSchemaBuilder.optional 110 | builder.field(fieldName, tsSchemaBuilder.build) 111 | 112 | case _ => 113 | logger.warn("JDBC type {} not currently supported", sqlType) 114 | } 115 | } 116 | 117 | private def convertFieldValue(resultSet: ResultSet, col: Int, colType: Int, struct: Struct, fieldName: String) = { 118 | val colValue = colType match { 119 | case Types.BOOLEAN => resultSet.getBoolean(col) 120 | 121 | case Types.BIT | Types.TINYINT => resultSet.getByte(col) 122 | 123 | case Types.SMALLINT => resultSet.getShort(col) 124 | 125 | case Types.INTEGER => resultSet.getInt(col) 126 | 127 | case Types.BIGINT => resultSet.getLong(col) 128 | 129 | case Types.REAL => resultSet.getFloat(col) 130 | 131 | case Types.FLOAT | Types.DOUBLE => resultSet.getDouble(col) 132 | 133 | case Types.NUMERIC | Types.DECIMAL => resultSet.getBigDecimal(col) 134 | 135 | case Types.CHAR | Types.VARCHAR | Types.LONGVARCHAR => resultSet.getString(col) 136 | 137 | case Types.NCHAR | Types.NVARCHAR | Types.LONGNVARCHAR => resultSet.getNString(col) 138 | 139 | case Types.CLOB | Types.NCLOB => 140 | val clob = if (colType == Types.CLOB) resultSet.getClob(col) else resultSet.getNClob(col) 141 | val bytes = 142 | if(clob == null) null 143 | else if(clob.length > Integer.MAX_VALUE) { 144 | logger.warn("Can't process CLOBs longer than Integer.MAX_VALUE") 145 | throw new IOException("Can't process CLOBs longer than Integer.MAX_VALUE") 146 | } 147 | else clob.getSubString(1, clob.length.toInt) 148 | if(clob != null) clob.free() 149 | bytes 150 | 151 | case Types.DATALINK => 152 | val url = resultSet.getURL(col) 153 | if (url != null) url.toString else null 154 | 155 | case Types.SQLXML => 156 | val xml = resultSet.getSQLXML(col) 157 | if (xml != null) xml.getString else null 158 | 159 | case Types.BINARY | Types.VARBINARY | Types.LONGVARBINARY => resultSet.getBytes(col) 160 | 161 | case Types.BLOB => 162 | val blob = resultSet.getBlob(col) 163 | val bytes = 164 | if (blob == null) null 165 | else if (blob.length > Integer.MAX_VALUE) { 166 | logger.warn("Can't process BLOBs longer than Integer.MAX_VALUE") 167 | throw new IOException("Can't process BLOBs longer than Integer.MAX_VALUE") 168 | } 169 | else blob.getBytes(1, blob.length.toInt) 170 | if(blob != null) blob.free() 171 | bytes 172 | 173 | case Types.DATE => resultSet.getDate(col, UTC_CALENDAR) 174 | 175 | case Types.TIME => resultSet.getTime(col, UTC_CALENDAR) 176 | 177 | case Types.TIMESTAMP => resultSet.getTimestamp(col, UTC_CALENDAR) 178 | 179 | case _ => null 180 | } 181 | 182 | struct.put(fieldName, if (resultSet.wasNull) null else colValue) 183 | } 184 | } -------------------------------------------------------------------------------- /src/main/scala/com/agoda/kafka/connector/jdbc/utils/Version.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.utils 2 | 3 | import com.agoda.BuildInfo 4 | 5 | object Version { 6 | lazy val getVersion: String = BuildInfo.version 7 | } 8 | -------------------------------------------------------------------------------- /src/test/scala/com/agoda/kafka/connector/jdbc/JdbcSourceConnectorConfigTest.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc 2 | 3 | import com.agoda.kafka.connector.jdbc.models.Mode.TimestampIncrementingMode 4 | import org.scalatest.{Matchers, WordSpec} 5 | 6 | class JdbcSourceConnectorConfigTest extends WordSpec with Matchers { 7 | import JdbcSourceConnectorConfigTestData._ 8 | 9 | "JDBC Source Connector Config" should { 10 | 11 | "throw exception if any mandatory configuration is missing" in { 12 | val properties = Map(connectionUrlProperty, incrementingModeProperty, topicProperty, 13 | pollIntervalProperty, batchVariableNameProperty, incrementingVariableNameConfig, incrementingFieldNameConfig) 14 | 15 | the [IllegalArgumentException] thrownBy new JdbcSourceConnectorConfig(properties).getClass 16 | } 17 | 18 | "create JDBC source configuration for incrementing mode" in { 19 | val properties = Map(connectionUrlProperty, incrementingModeProperty, storedProcedureProperty, topicProperty, 20 | pollIntervalProperty, batchVariableNameProperty, incrementingVariableNameConfig, incrementingFieldNameConfig) 21 | 22 | new JdbcSourceConnectorConfig(properties).getClass shouldEqual classOf[JdbcSourceConnectorConfig] 23 | } 24 | 25 | "throw exception if any configuration for incrementing mode is missing" in { 26 | val properties = Map(connectionUrlProperty, incrementingModeProperty, storedProcedureProperty, topicProperty, 27 | pollIntervalProperty, batchVariableNameProperty, incrementingVariableNameConfig) 28 | 29 | the [IllegalArgumentException] thrownBy new JdbcSourceConnectorConfig(properties).getClass 30 | } 31 | 32 | "create JDBC source configuration for timestamp mode" in { 33 | val properties = Map(connectionUrlProperty, timestampModeProperty, storedProcedureProperty, topicProperty, 34 | pollIntervalProperty, batchVariableNameProperty, timestampVariableNameConfig, timestampFieldNameConfig) 35 | 36 | new JdbcSourceConnectorConfig(properties).getClass shouldEqual classOf[JdbcSourceConnectorConfig] 37 | } 38 | 39 | "throw exception if any configuration for timestamp mode is missing" in { 40 | val properties = Map(connectionUrlProperty, timestampModeProperty, storedProcedureProperty, topicProperty, 41 | pollIntervalProperty, batchVariableNameProperty, timestampFieldNameConfig) 42 | 43 | the [IllegalArgumentException] thrownBy new JdbcSourceConnectorConfig(properties).getClass 44 | } 45 | 46 | "create JDBC source configuration for timestamp+incrementing mode" in { 47 | val properties = Map(connectionUrlProperty, timestampIncrementingModeProperty, storedProcedureProperty, 48 | topicProperty, pollIntervalProperty, batchVariableNameProperty, timestampVariableNameConfig, timestampFieldNameConfig, 49 | incrementingVariableNameConfig, incrementingFieldNameConfig) 50 | 51 | new JdbcSourceConnectorConfig(properties).getClass shouldEqual classOf[JdbcSourceConnectorConfig] 52 | } 53 | 54 | "throw exception if any configuration for timestamp+incrementing mode is missing" in { 55 | val properties = Map(connectionUrlProperty, timestampIncrementingModeProperty, storedProcedureProperty, 56 | topicProperty, pollIntervalProperty, batchVariableNameProperty, timestampFieldNameConfig, 57 | incrementingVariableNameConfig) 58 | 59 | the [IllegalArgumentException] thrownBy new JdbcSourceConnectorConfig(properties).getClass 60 | } 61 | 62 | "get all properties from configuration" in { 63 | val properties = Map(connectionUrlProperty, timestampIncrementingModeProperty, storedProcedureProperty, 64 | topicProperty, pollIntervalProperty, batchSizeProperty, batchVariableNameProperty, timestampVariableNameConfig, 65 | timestampFieldNameConfig, timestampOffsetConfig, incrementingVariableNameConfig, incrementingFieldNameConfig, 66 | incrementingOffsetConfig, keyFieldConfig) 67 | 68 | val configuration = new JdbcSourceConnectorConfig(properties) 69 | 70 | configuration.getConnectionUrl shouldBe "test-connection" 71 | configuration.getMode shouldBe TimestampIncrementingMode 72 | configuration.getStoredProcedureName shouldBe "test-procedure" 73 | configuration.getTopic shouldBe "test-topic" 74 | configuration.getPollInterval shouldBe 100 75 | configuration.getMaxBatchSize shouldBe 1000 76 | configuration.getMaxBatchSizeVariableName shouldBe "batch" 77 | configuration.getTimestampVariableName shouldBe Some("time") 78 | configuration.getTimestampFieldName shouldBe Some("time") 79 | configuration.getIncrementingVariableName shouldBe Some("id") 80 | configuration.getIncrementingFieldName shouldBe Some("id") 81 | configuration.getTimestampOffset shouldBe 946684800000L 82 | configuration.getIncrementingOffset shouldBe 5L 83 | configuration.getKeyField shouldBe Some("test-key") 84 | } 85 | 86 | "get optional properties with default values from configuration" in { 87 | val properties = Map(connectionUrlProperty, timestampIncrementingModeProperty, storedProcedureProperty, 88 | topicProperty, batchVariableNameProperty, timestampVariableNameConfig, timestampFieldNameConfig, 89 | incrementingVariableNameConfig, incrementingFieldNameConfig) 90 | 91 | val configuration = new JdbcSourceConnectorConfig(properties) 92 | 93 | configuration.getPollInterval shouldBe 5000 94 | configuration.getMaxBatchSize shouldBe 100 95 | configuration.getTimestampOffset shouldBe 0L 96 | configuration.getIncrementingOffset shouldBe 0L 97 | } 98 | } 99 | } 100 | 101 | object JdbcSourceConnectorConfigTestData { 102 | val connectionUrlProperty: (String, String) = "connection.url" -> "test-connection" 103 | val timestampModeProperty: (String, String) = "mode" -> "timestamp" 104 | val incrementingModeProperty: (String, String) = "mode" -> "incrementing" 105 | val timestampIncrementingModeProperty: (String, String) = "mode" -> "timestamp+incrementing" 106 | val storedProcedureProperty: (String, String) = "stored-procedure.name" -> "test-procedure" 107 | val topicProperty: (String, String) = "topic" -> "test-topic" 108 | val pollIntervalProperty: (String, String) = "poll.interval.ms" -> "100" 109 | val batchSizeProperty: (String, String) = "batch.max.records" -> "1000" 110 | val batchVariableNameProperty: (String, String) = "batch.max.rows.variable.name" -> "batch" 111 | val timestampVariableNameConfig: (String, String) = "timestamp.variable.name" -> "time" 112 | val timestampFieldNameConfig: (String, String) = "timestamp.field.name" -> "time" 113 | val timestampOffsetConfig: (String, String) = "timestamp.offset" -> "2000-01-01 00:00:00" 114 | val incrementingVariableNameConfig: (String, String) = "incrementing.variable.name" -> "id" 115 | val incrementingFieldNameConfig: (String, String) = "incrementing.field.name" -> "id" 116 | val incrementingOffsetConfig: (String, String) = "incrementing.offset" -> "5" 117 | val keyFieldConfig: (String, String) = "key.field.name" -> "test-key" 118 | } 119 | -------------------------------------------------------------------------------- /src/test/scala/com/agoda/kafka/connector/jdbc/models/DatabaseProductTest.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.models 2 | 3 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL} 4 | import org.scalatest.{Matchers, WordSpec} 5 | 6 | class DatabaseProductTest extends WordSpec with Matchers { 7 | 8 | "module" should { 9 | "convert DatabaseProduct to its string representation" in { 10 | DatabaseProduct.MySQL.entryName shouldEqual "MySQL" 11 | DatabaseProduct.MsSQL.entryName shouldEqual "Microsoft SQL Server" 12 | } 13 | 14 | "convert string to corresponding DatabaseProduct representation" in { 15 | DatabaseProduct.withName("MySQL") shouldBe MySQL 16 | DatabaseProduct.withName("Microsoft SQL Server") shouldBe MsSQL 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/test/scala/com/agoda/kafka/connector/jdbc/models/ModeTest.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.models 2 | 3 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampIncrementingMode, TimestampMode} 4 | import org.scalatest.{Matchers, WordSpec} 5 | 6 | class ModeTest extends WordSpec with Matchers { 7 | 8 | "module" should { 9 | "convert Mode to its string representation" in { 10 | Mode.TimestampMode.entryName shouldEqual "timestamp" 11 | Mode.IncrementingMode.entryName shouldEqual "incrementing" 12 | Mode.TimestampIncrementingMode.entryName shouldEqual "timestamp+incrementing" 13 | } 14 | 15 | "convert string to corresponding Mode representation" in { 16 | Mode.withName("timestamp") shouldBe TimestampMode 17 | Mode.withName("incrementing") shouldBe IncrementingMode 18 | Mode.withName("timestamp+incrementing") shouldBe TimestampIncrementingMode 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/test/scala/com/agoda/kafka/connector/jdbc/services/DataServiceTest.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.services 2 | 3 | import java.sql.{Connection, PreparedStatement, ResultSet, ResultSetMetaData} 4 | 5 | import com.agoda.kafka.connector.jdbc.utils.DataConverter 6 | import org.apache.kafka.connect.data.Schema 7 | import org.apache.kafka.connect.source.SourceRecord 8 | import org.scalatest.mockito.MockitoSugar 9 | import org.mockito.Mockito._ 10 | import org.scalatest.{Matchers, WordSpec} 11 | 12 | import scala.concurrent.duration._ 13 | import scala.util.Success 14 | 15 | class DataServiceTest extends WordSpec with Matchers with MockitoSugar { 16 | 17 | "Data Service" should { 18 | 19 | val spName = "stored-procedure" 20 | val connection = mock[Connection] 21 | val converter = mock[DataConverter] 22 | val sourceRecord1 = mock[SourceRecord] 23 | val sourceRecord2 = mock[SourceRecord] 24 | val resultSet = mock[ResultSet] 25 | val resultSetMetadata = mock[ResultSetMetaData] 26 | val preparedStatement = mock[PreparedStatement] 27 | val schema = mock[Schema] 28 | 29 | val dataService = new DataService { 30 | 31 | override def storedProcedureName: String = spName 32 | 33 | override protected def createPreparedStatement(connection: Connection) = Success(preparedStatement) 34 | 35 | override protected def extractRecords(resultSet: ResultSet, schema: Schema) = Success(Seq(sourceRecord1, sourceRecord2)) 36 | 37 | override def dataConverter: DataConverter = converter 38 | } 39 | 40 | "get records" in { 41 | doNothing().when(preparedStatement).setQueryTimeout(1) 42 | when(preparedStatement.executeQuery).thenReturn(resultSet) 43 | when(resultSet.getMetaData).thenReturn(resultSetMetadata) 44 | when(converter.convertSchema(spName, resultSetMetadata)).thenReturn(Success(schema)) 45 | 46 | dataService.getRecords(connection, 1.second) shouldBe Success(Seq(sourceRecord1, sourceRecord2)) 47 | 48 | verify(preparedStatement).setQueryTimeout(1) 49 | verify(preparedStatement).executeQuery 50 | verify(resultSet).getMetaData 51 | verify(converter).convertSchema(spName, resultSetMetadata) 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/test/scala/com/agoda/kafka/connector/jdbc/services/IdBasedDataServiceTest.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.services 2 | 3 | import java.sql.{Connection, PreparedStatement, ResultSet} 4 | 5 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants 6 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL} 7 | import com.agoda.kafka.connector.jdbc.models.Mode.IncrementingMode 8 | import com.agoda.kafka.connector.jdbc.utils.DataConverter 9 | import org.apache.kafka.connect.data.{Field, Schema, Struct} 10 | import org.apache.kafka.connect.source.SourceRecord 11 | import org.mockito.Mockito._ 12 | import org.scalatest.mockito.MockitoSugar 13 | import org.scalatest.{Matchers, WordSpec} 14 | 15 | import scala.collection.JavaConverters._ 16 | import scala.collection.mutable.ListBuffer 17 | import scala.util.Success 18 | 19 | class IdBasedDataServiceTest extends WordSpec with Matchers with MockitoSugar { 20 | 21 | "ID Based Data Service" should { 22 | 23 | val dataConverter = mock[DataConverter] 24 | 25 | val idBasedDataServiceMssql = 26 | IdBasedDataService( 27 | databaseProduct = MsSQL, 28 | storedProcedureName = "stored-procedure", 29 | batchSize = 100, 30 | batchSizeVariableName = "batch-size-variable", 31 | incrementingVariableName = "incrementing-variable", 32 | incrementingOffset = 0L, 33 | incrementingFieldName = "id", 34 | topic = "id-based-data-topic", 35 | keyFieldOpt = None, 36 | dataConverter = dataConverter 37 | ) 38 | 39 | val idBasedDataServiceMysql = 40 | IdBasedDataService( 41 | databaseProduct = MySQL, 42 | storedProcedureName = "stored-procedure", 43 | batchSize = 100, 44 | batchSizeVariableName = "batch-size-variable", 45 | incrementingVariableName = "incrementing-variable", 46 | incrementingOffset = 0L, 47 | incrementingFieldName = "id", 48 | topic = "id-based-data-topic", 49 | keyFieldOpt = None, 50 | dataConverter = dataConverter 51 | ) 52 | 53 | "create correct prepared statement for Mssql" in { 54 | val connection = mock[Connection] 55 | val statement = mock[PreparedStatement] 56 | 57 | when(connection.prepareStatement("EXECUTE stored-procedure @incrementing-variable = ?, @batch-size-variable = ?")).thenReturn(statement) 58 | doNothing().when(statement).setObject(1, 0L) 59 | doNothing().when(statement).setObject(2, 100) 60 | 61 | idBasedDataServiceMssql.createPreparedStatement(connection) 62 | 63 | verify(connection).prepareStatement("EXECUTE stored-procedure @incrementing-variable = ?, @batch-size-variable = ?") 64 | verify(statement).setObject(1, 0L) 65 | verify(statement).setObject(2, 100) 66 | } 67 | 68 | "create correct prepared statement for Mysql" in { 69 | val connection = mock[Connection] 70 | val statement = mock[PreparedStatement] 71 | 72 | when(connection.prepareStatement("CALL stored-procedure (@incrementing-variable := ?, @batch-size-variable := ?)")).thenReturn(statement) 73 | doNothing().when(statement).setObject(1, 0L) 74 | doNothing().when(statement).setObject(2, 100) 75 | 76 | idBasedDataServiceMysql.createPreparedStatement(connection) 77 | 78 | verify(connection).prepareStatement("CALL stored-procedure (@incrementing-variable := ?, @batch-size-variable := ?)") 79 | verify(statement).setObject(1, 0L) 80 | verify(statement).setObject(2, 100) 81 | } 82 | 83 | "create correct string representation" in { 84 | idBasedDataServiceMssql.toString shouldBe 85 | s""" 86 | |{ 87 | | "name" : "IdBasedDataService" 88 | | "mode" : "incrementing" 89 | | "stored-procedure.name" : "stored-procedure" 90 | |} 91 | """.stripMargin 92 | } 93 | 94 | "extract records" in { 95 | val resultSet = mock[ResultSet] 96 | val schema = mock[Schema] 97 | val field = mock[Field] 98 | val struct = mock[Struct] 99 | 100 | when(schema.field("id")).thenReturn(field) 101 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA) 102 | when(resultSet.next()).thenReturn(true, true, false) 103 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 104 | when(struct.getInt32("id")).thenReturn(1, 2) 105 | 106 | 107 | idBasedDataServiceMssql.extractRecords(resultSet, schema).toString shouldBe 108 | Success( 109 | ListBuffer( 110 | new SourceRecord( 111 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 112 | Map(IncrementingMode.entryName -> 1).asJava, "id-based-data-topic", schema, struct 113 | ), 114 | new SourceRecord( 115 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 116 | Map(IncrementingMode.entryName -> 2).asJava, "id-based-data-topic", schema, struct 117 | ) 118 | ) 119 | ).toString 120 | idBasedDataServiceMssql.incrementingOffset shouldBe 2 121 | } 122 | 123 | "return nothing when record ids are smaller than or equal to the offset id" in { 124 | val idBasedDataServiceWithLargerOffsetMssql = idBasedDataServiceMssql.copy(incrementingOffset = 3L) 125 | val resultSet = mock[ResultSet] 126 | val schema = mock[Schema] 127 | val field = mock[Field] 128 | val struct = mock[Struct] 129 | 130 | when(schema.field("id")).thenReturn(field) 131 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA) 132 | when(resultSet.next()).thenReturn(true, true, true, false) 133 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 134 | when(struct.getInt32("id")).thenReturn(1, 2, 3) 135 | 136 | idBasedDataServiceWithLargerOffsetMssql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer()) 137 | idBasedDataServiceWithLargerOffsetMssql.incrementingOffset shouldBe 3L 138 | } 139 | 140 | "extract records with key" in { 141 | val idBasedDataServiceWithKeyMysql = idBasedDataServiceMysql.copy(keyFieldOpt = Some("key")) 142 | val resultSet = mock[ResultSet] 143 | val schema = mock[Schema] 144 | val field = mock[Field] 145 | val struct = mock[Struct] 146 | 147 | when(schema.field("id")).thenReturn(field) 148 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA) 149 | when(resultSet.next()).thenReturn(true, true, false) 150 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 151 | when(struct.getInt32("id")).thenReturn(1, 2) 152 | when(struct.get("key")).thenReturn("key-1", "key-2") 153 | 154 | 155 | idBasedDataServiceWithKeyMysql.extractRecords(resultSet, schema).toString shouldBe 156 | Success( 157 | ListBuffer( 158 | new SourceRecord( 159 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 160 | Map(IncrementingMode.entryName -> 1).asJava, "id-based-data-topic", null, schema, "key-1", schema, struct 161 | ), 162 | new SourceRecord( 163 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 164 | Map(IncrementingMode.entryName -> 2).asJava, "id-based-data-topic", null, schema, "key-2", schema, struct 165 | ) 166 | ) 167 | ).toString 168 | } 169 | 170 | "returns no record when id field is not of type integer" in { 171 | val resultSet = mock[ResultSet] 172 | val schema = mock[Schema] 173 | val field = mock[Field] 174 | val struct = mock[Struct] 175 | 176 | when(schema.field("id")).thenReturn(field) 177 | when(field.schema()).thenReturn(Schema.STRING_SCHEMA) 178 | when(resultSet.next()).thenReturn(true, false) 179 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 180 | 181 | idBasedDataServiceMysql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer()) 182 | } 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /src/test/scala/com/agoda/kafka/connector/jdbc/services/TimeBasedDataServiceTest.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.services 2 | 3 | import java.sql._ 4 | import java.util.{GregorianCalendar, TimeZone} 5 | 6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants 7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL} 8 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampMode} 9 | import com.agoda.kafka.connector.jdbc.utils.DataConverter 10 | import org.apache.kafka.connect.data.{Field, Schema, Struct} 11 | import org.apache.kafka.connect.source.SourceRecord 12 | import org.mockito.Mockito._ 13 | import org.scalatest.mockito.MockitoSugar 14 | import org.scalatest.{Matchers, WordSpec} 15 | 16 | import scala.collection.JavaConverters._ 17 | import scala.collection.mutable.ListBuffer 18 | import scala.util.Success 19 | 20 | class TimeBasedDataServiceTest extends WordSpec with Matchers with MockitoSugar { 21 | 22 | "Time Based Data Service" should { 23 | 24 | val dataConverter = mock[DataConverter] 25 | val UTC_Calendar = new GregorianCalendar(TimeZone.getTimeZone("UTC")) 26 | 27 | val timeBasedDataServiceMssql = 28 | TimeBasedDataService( 29 | databaseProduct = MsSQL, 30 | storedProcedureName = "stored-procedure", 31 | batchSize = 100, 32 | batchSizeVariableName = "batch-size-variable", 33 | timestampVariableName = "timestamp-variable", 34 | timestampOffset = 0L, 35 | timestampFieldName = "time", 36 | topic = "time-based-data-topic", 37 | keyFieldOpt = None, 38 | dataConverter = dataConverter, 39 | calendar = UTC_Calendar 40 | ) 41 | 42 | val timeBasedDataServiceMysql = 43 | TimeBasedDataService( 44 | databaseProduct = MySQL, 45 | storedProcedureName = "stored-procedure", 46 | batchSize = 100, 47 | batchSizeVariableName = "batch-size-variable", 48 | timestampVariableName = "timestamp-variable", 49 | timestampOffset = 0L, 50 | timestampFieldName = "time", 51 | topic = "time-based-data-topic", 52 | keyFieldOpt = None, 53 | dataConverter = dataConverter, 54 | calendar = UTC_Calendar 55 | ) 56 | 57 | val timestamp = new Timestamp(0L) 58 | 59 | "create correct prepared statement for Mssql" in { 60 | val connection = mock[Connection] 61 | val statement = mock[PreparedStatement] 62 | 63 | when(connection.prepareStatement("EXECUTE stored-procedure @timestamp-variable = ?, @batch-size-variable = ?")).thenReturn(statement) 64 | doNothing().when(statement).setTimestamp(1, timestamp, UTC_Calendar) 65 | doNothing().when(statement).setObject(2, 100) 66 | 67 | timeBasedDataServiceMssql.createPreparedStatement(connection) 68 | 69 | verify(connection).prepareStatement("EXECUTE stored-procedure @timestamp-variable = ?, @batch-size-variable = ?") 70 | verify(statement).setTimestamp(1, timestamp, UTC_Calendar) 71 | verify(statement).setObject(2, 100) 72 | } 73 | 74 | "create correct prepared statement for Mysql" in { 75 | val connection = mock[Connection] 76 | val statement = mock[PreparedStatement] 77 | 78 | when(connection.prepareStatement("CALL stored-procedure (@timestamp-variable := ?, @batch-size-variable := ?)")).thenReturn(statement) 79 | doNothing().when(statement).setTimestamp(1, timestamp, UTC_Calendar) 80 | doNothing().when(statement).setObject(2, 100) 81 | 82 | timeBasedDataServiceMysql.createPreparedStatement(connection) 83 | 84 | verify(connection).prepareStatement("CALL stored-procedure (@timestamp-variable := ?, @batch-size-variable := ?)") 85 | verify(statement).setTimestamp(1, timestamp, UTC_Calendar) 86 | verify(statement).setObject(2, 100) 87 | } 88 | 89 | "create correct string representation" in { 90 | timeBasedDataServiceMssql.toString shouldBe 91 | s""" 92 | |{ 93 | | "name" : "TimeBasedDataService" 94 | | "mode" : "timestamp" 95 | | "stored-procedure.name" : "stored-procedure" 96 | |} 97 | """.stripMargin 98 | } 99 | 100 | "extract records" in { 101 | val resultSet = mock[ResultSet] 102 | val schema = mock[Schema] 103 | val struct = mock[Struct] 104 | 105 | when(resultSet.next()).thenReturn(true, true, false) 106 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 107 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L)) 108 | 109 | timeBasedDataServiceMssql.extractRecords(resultSet, schema).toString shouldBe 110 | Success( 111 | ListBuffer( 112 | new SourceRecord( 113 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 114 | Map(TimestampMode.entryName -> 1L).asJava, "time-based-data-topic", schema, struct 115 | ), 116 | new SourceRecord( 117 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 118 | Map(TimestampMode.entryName -> 2L).asJava, "time-based-data-topic", schema, struct 119 | ) 120 | ) 121 | ).toString 122 | timeBasedDataServiceMssql.timestampOffset shouldBe 2L 123 | } 124 | 125 | "return nothing when the record timestamps are smaller than or equal to the offset timestamp" in { 126 | val timeBasedDataServiceWithLargerOffsetMssql = timeBasedDataServiceMssql.copy(timestampOffset = 3L) 127 | val resultSet = mock[ResultSet] 128 | val schema = mock[Schema] 129 | val struct = mock[Struct] 130 | 131 | when(resultSet.next()).thenReturn(true, true, true, false) 132 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 133 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L), new Date(3L)) 134 | 135 | timeBasedDataServiceWithLargerOffsetMssql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer()) 136 | timeBasedDataServiceWithLargerOffsetMssql.timestampOffset shouldBe 3L 137 | } 138 | 139 | "extract records with key" in { 140 | val timeBasedDataServiceWithKeyMysql = timeBasedDataServiceMysql.copy(keyFieldOpt = Some("key")) 141 | val resultSet = mock[ResultSet] 142 | val schema = mock[Schema] 143 | val struct = mock[Struct] 144 | 145 | when(resultSet.next()).thenReturn(true, true, false) 146 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 147 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L)) 148 | when(struct.get("key")).thenReturn("key-1", "key-2") 149 | 150 | timeBasedDataServiceWithKeyMysql.extractRecords(resultSet, schema).toString shouldBe 151 | Success( 152 | ListBuffer( 153 | new SourceRecord( 154 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 155 | Map(TimestampMode.entryName -> 1L).asJava, "time-based-data-topic", null, schema, "key-1", schema, struct 156 | ), 157 | new SourceRecord( 158 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 159 | Map(TimestampMode.entryName -> 2L).asJava, "time-based-data-topic", null, schema, "key-2", schema, struct 160 | ) 161 | ) 162 | ).toString 163 | } 164 | } 165 | } 166 | 167 | -------------------------------------------------------------------------------- /src/test/scala/com/agoda/kafka/connector/jdbc/services/TimeIdBasedDataServiceTest.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.services 2 | 3 | import java.sql._ 4 | import java.util.{GregorianCalendar, TimeZone} 5 | 6 | import com.agoda.kafka.connector.jdbc.JdbcSourceConnectorConstants 7 | import com.agoda.kafka.connector.jdbc.models.DatabaseProduct.{MsSQL, MySQL} 8 | import com.agoda.kafka.connector.jdbc.models.Mode.{IncrementingMode, TimestampMode} 9 | import com.agoda.kafka.connector.jdbc.utils.DataConverter 10 | import org.apache.kafka.connect.data.{Field, Schema, Struct} 11 | import org.apache.kafka.connect.source.SourceRecord 12 | import org.mockito.Mockito._ 13 | import org.scalatest.mockito.MockitoSugar 14 | import org.scalatest.{Matchers, WordSpec} 15 | 16 | import scala.collection.JavaConverters._ 17 | import scala.collection.mutable.ListBuffer 18 | import scala.util.Success 19 | 20 | class TimeIdBasedDataServiceTest extends WordSpec with Matchers with MockitoSugar { 21 | 22 | "Time ID Based Data Service" should { 23 | 24 | val dataConverter = mock[DataConverter] 25 | 26 | val UTC_CALENDAR = new GregorianCalendar(TimeZone.getTimeZone("UTC")) 27 | 28 | val timeIdBasedDataServiceMssql = 29 | TimeIdBasedDataService( 30 | databaseProduct = MsSQL, 31 | storedProcedureName = "stored-procedure", 32 | batchSize = 100, 33 | batchSizeVariableName = "batch-size-variable", 34 | timestampVariableName = "timestamp-variable", 35 | timestampOffset = 0L, 36 | timestampFieldName = "time", 37 | incrementingVariableName = "incrementing-variable", 38 | incrementingOffset = 0L, 39 | incrementingFieldName = "id", 40 | topic = "time-id-based-data-topic", 41 | keyFieldOpt = None, 42 | dataConverter = dataConverter, 43 | calendar = UTC_CALENDAR 44 | ) 45 | 46 | val timeIdBasedDataServiceMysql = 47 | TimeIdBasedDataService( 48 | databaseProduct = MySQL, 49 | storedProcedureName = "stored-procedure", 50 | batchSize = 100, 51 | batchSizeVariableName = "batch-size-variable", 52 | timestampVariableName = "timestamp-variable", 53 | timestampOffset = 0L, 54 | timestampFieldName = "time", 55 | incrementingVariableName = "incrementing-variable", 56 | incrementingOffset = 0L, 57 | incrementingFieldName = "id", 58 | topic = "time-id-based-data-topic", 59 | keyFieldOpt = None, 60 | dataConverter = dataConverter, 61 | calendar = UTC_CALENDAR 62 | ) 63 | 64 | val timestamp = new Timestamp(0L) 65 | 66 | "create correct prepared statement for Mssql" in { 67 | val connection = mock[Connection] 68 | val statement = mock[PreparedStatement] 69 | 70 | when(connection.prepareStatement("EXECUTE stored-procedure @timestamp-variable = ?, @incrementing-variable = ?, @batch-size-variable = ?")).thenReturn(statement) 71 | doNothing().when(statement).setTimestamp(1, timestamp, UTC_CALENDAR) 72 | doNothing().when(statement).setObject(2, 0L) 73 | doNothing().when(statement).setObject(3, 100) 74 | 75 | timeIdBasedDataServiceMssql.createPreparedStatement(connection) 76 | 77 | verify(connection).prepareStatement("EXECUTE stored-procedure @timestamp-variable = ?, @incrementing-variable = ?, @batch-size-variable = ?") 78 | verify(statement).setTimestamp(1, timestamp, UTC_CALENDAR) 79 | verify(statement).setObject(2, 0L) 80 | verify(statement).setObject(3, 100) 81 | } 82 | 83 | "create correct prepared statement for Mysql" in { 84 | val connection = mock[Connection] 85 | val statement = mock[PreparedStatement] 86 | 87 | when(connection.prepareStatement("CALL stored-procedure (@timestamp-variable := ?, @incrementing-variable := ?, @batch-size-variable := ?)")).thenReturn(statement) 88 | doNothing().when(statement).setTimestamp(1, timestamp, UTC_CALENDAR) 89 | doNothing().when(statement).setObject(2, 0L) 90 | doNothing().when(statement).setObject(3, 100) 91 | 92 | timeIdBasedDataServiceMysql.createPreparedStatement(connection) 93 | 94 | verify(connection).prepareStatement("CALL stored-procedure (@timestamp-variable := ?, @incrementing-variable := ?, @batch-size-variable := ?)") 95 | verify(statement).setTimestamp(1, timestamp, UTC_CALENDAR) 96 | verify(statement).setObject(2, 0L) 97 | verify(statement).setObject(3, 100) 98 | } 99 | 100 | "create correct string representation" in { 101 | timeIdBasedDataServiceMssql.toString shouldBe 102 | s""" 103 | |{ 104 | | "name" : "TimeIdBasedDataService" 105 | | "mode" : "timestamp+incrementing" 106 | | "stored-procedure.name" : "stored-procedure" 107 | |} 108 | """.stripMargin 109 | } 110 | 111 | "extract records" in { 112 | val resultSet = mock[ResultSet] 113 | val schema = mock[Schema] 114 | val field = mock[Field] 115 | val struct = mock[Struct] 116 | 117 | when(schema.field("id")).thenReturn(field) 118 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA) 119 | when(resultSet.next()).thenReturn(true, true, false) 120 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 121 | when(struct.getInt32("id")).thenReturn(1, 2) 122 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L)) 123 | 124 | timeIdBasedDataServiceMssql.extractRecords(resultSet, schema).toString shouldBe 125 | Success( 126 | ListBuffer( 127 | new SourceRecord( 128 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 129 | Map(TimestampMode.entryName -> 1L, IncrementingMode.entryName -> 1).asJava, 130 | "time-id-based-data-topic", schema, struct 131 | ), 132 | new SourceRecord( 133 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 134 | Map(TimestampMode.entryName -> 2L, IncrementingMode.entryName -> 2).asJava, 135 | "time-id-based-data-topic", schema, struct 136 | ) 137 | ) 138 | ).toString 139 | timeIdBasedDataServiceMssql.timestampOffset shouldBe 2L 140 | timeIdBasedDataServiceMssql.incrementingOffset shouldBe 2 141 | } 142 | 143 | "return nothing when record ids and timestamps are smaller than offset id" in { 144 | val timeIdBasedDataServiceWithLargerOffsetMssql = timeIdBasedDataServiceMssql.copy(incrementingOffset = 3, timestampOffset = 3L) 145 | val resultSet = mock[ResultSet] 146 | val schema = mock[Schema] 147 | val field = mock[Field] 148 | val struct = mock[Struct] 149 | 150 | when(schema.field("id")).thenReturn(field) 151 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA) 152 | when(resultSet.next()).thenReturn(true, true, false) 153 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 154 | when(struct.getInt32("id")).thenReturn(1, 2) 155 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L)) 156 | 157 | timeIdBasedDataServiceWithLargerOffsetMssql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer()) 158 | 159 | timeIdBasedDataServiceWithLargerOffsetMssql.timestampOffset shouldBe 3L 160 | timeIdBasedDataServiceWithLargerOffsetMssql.incrementingOffset shouldBe 3 161 | } 162 | 163 | "return record when the record timestamps are equals but record ids are than offset id" in { 164 | val timeIdBasedDataServiceWithLargerOffsetMssql = timeIdBasedDataServiceMssql.copy(incrementingOffset = 3, timestampOffset = 3L) 165 | val resultSet = mock[ResultSet] 166 | val schema = mock[Schema] 167 | val field = mock[Field] 168 | val struct = mock[Struct] 169 | 170 | when(schema.field("id")).thenReturn(field) 171 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA) 172 | when(resultSet.next()).thenReturn(true, true, false) 173 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 174 | when(struct.getInt32("id")).thenReturn(4, 5) 175 | when(struct.get("time")).thenReturn(new Date(3L), new Date(3L)) 176 | 177 | timeIdBasedDataServiceWithLargerOffsetMssql.extractRecords(resultSet, schema).toString shouldBe 178 | Success( 179 | ListBuffer( 180 | new SourceRecord( 181 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 182 | Map(TimestampMode.entryName -> 3L, IncrementingMode.entryName -> 4).asJava, 183 | "time-id-based-data-topic", schema, struct 184 | ), 185 | new SourceRecord( 186 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 187 | Map(TimestampMode.entryName -> 3L, IncrementingMode.entryName -> 5).asJava, 188 | "time-id-based-data-topic", schema, struct 189 | ) 190 | ) 191 | ).toString 192 | 193 | timeIdBasedDataServiceWithLargerOffsetMssql.timestampOffset shouldBe 3L 194 | timeIdBasedDataServiceWithLargerOffsetMssql.incrementingOffset shouldBe 5L 195 | } 196 | 197 | "extract records with key" in { 198 | val timeIdBasedDataServiceWithKeyMysql = timeIdBasedDataServiceMysql.copy(keyFieldOpt = Some("key")) 199 | val resultSet = mock[ResultSet] 200 | val schema = mock[Schema] 201 | val field = mock[Field] 202 | val struct = mock[Struct] 203 | 204 | when(schema.field("id")).thenReturn(field) 205 | when(field.schema()).thenReturn(Schema.INT32_SCHEMA) 206 | when(resultSet.next()).thenReturn(true, true, false) 207 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 208 | when(struct.getInt32("id")).thenReturn(1, 2) 209 | when(struct.get("time")).thenReturn(new Date(1L), new Date(2L)) 210 | when(struct.get("key")).thenReturn("key-1", "key-2") 211 | 212 | timeIdBasedDataServiceWithKeyMysql.extractRecords(resultSet, schema).toString shouldBe 213 | Success( 214 | ListBuffer( 215 | new SourceRecord( 216 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 217 | Map(TimestampMode.entryName -> 1L, IncrementingMode.entryName -> 1).asJava, 218 | "time-id-based-data-topic", null, schema, "key-1", schema, struct 219 | ), 220 | new SourceRecord( 221 | Map(JdbcSourceConnectorConstants.STORED_PROCEDURE_NAME_KEY -> "stored-procedure").asJava, 222 | Map(TimestampMode.entryName -> 2L, IncrementingMode.entryName -> 2).asJava, 223 | "time-id-based-data-topic", null, schema, "key-2", schema, struct 224 | ) 225 | ) 226 | ).toString 227 | } 228 | 229 | "returns no record when id field is not of type integer" in { 230 | val resultSet = mock[ResultSet] 231 | val schema = mock[Schema] 232 | val field = mock[Field] 233 | val struct = mock[Struct] 234 | 235 | when(schema.field("id")).thenReturn(field) 236 | when(field.schema()).thenReturn(Schema.STRING_SCHEMA) 237 | when(resultSet.next()).thenReturn(true, false) 238 | when(dataConverter.convertRecord(schema, resultSet)).thenReturn(Success(struct)) 239 | when(struct.get("time")).thenReturn(new Date(1L), Nil) 240 | 241 | timeIdBasedDataServiceMysql.extractRecords(resultSet, schema) shouldBe Success(ListBuffer()) 242 | } 243 | } 244 | } 245 | -------------------------------------------------------------------------------- /src/test/scala/com/agoda/kafka/connector/jdbc/utils/DataConverterTest.scala: -------------------------------------------------------------------------------- 1 | package com.agoda.kafka.connector.jdbc.utils 2 | 3 | import java.net.URL 4 | import java.math.BigDecimal 5 | import java.sql.{Blob, Clob, NClob, ResultSet, ResultSetMetaData, SQLXML, Types, Date => SqlDate, Time => SqlTime, Timestamp => SqlTimestamp} 6 | 7 | import org.apache.kafka.connect.data._ 8 | import org.apache.kafka.connect.errors.DataException 9 | import org.mockito.Mockito._ 10 | import org.mockito.ArgumentMatchers._ 11 | import org.scalatest.mockito.MockitoSugar 12 | import org.scalatest.{Matchers, WordSpec} 13 | 14 | import scala.util.{Failure, Success} 15 | 16 | class DataConverterTest extends WordSpec with Matchers with MockitoSugar { 17 | 18 | "Data Converter" should { 19 | 20 | val dataConverter = new DataConverter 21 | 22 | "create schema from result set metadata with non optional columns" in { 23 | val metaRS = mock[ResultSetMetaData] 24 | 25 | when(metaRS.getColumnCount).thenReturn(28) 26 | 27 | when(metaRS.getColumnName(1)).thenReturn("BOOLEAN") 28 | when(metaRS.getColumnName(2)).thenReturn("BIT") 29 | when(metaRS.getColumnName(3)).thenReturn("TINYINT") 30 | when(metaRS.getColumnName(4)).thenReturn("SMALLINT") 31 | when(metaRS.getColumnName(5)).thenReturn("INTEGER") 32 | when(metaRS.getColumnName(6)).thenReturn("BIGINT") 33 | when(metaRS.getColumnName(7)).thenReturn("REAL") 34 | when(metaRS.getColumnName(8)).thenReturn("FLOAT") 35 | when(metaRS.getColumnName(9)).thenReturn("DOUBLE") 36 | when(metaRS.getColumnName(10)).thenReturn("NUMERIC") 37 | when(metaRS.getColumnName(11)).thenReturn("DECIMAL") 38 | when(metaRS.getColumnName(12)).thenReturn("CHAR") 39 | when(metaRS.getColumnName(13)).thenReturn("VARCHAR") 40 | when(metaRS.getColumnName(14)).thenReturn("LONGVARCHAR") 41 | when(metaRS.getColumnName(15)).thenReturn("NCHAR") 42 | when(metaRS.getColumnName(16)).thenReturn("NVARCHAR") 43 | when(metaRS.getColumnName(17)).thenReturn("LONGNVARCHAR") 44 | when(metaRS.getColumnName(18)).thenReturn("CLOB") 45 | when(metaRS.getColumnName(19)).thenReturn("NCLOB") 46 | when(metaRS.getColumnName(20)).thenReturn("DATALINK") 47 | when(metaRS.getColumnName(21)).thenReturn("SQLXML") 48 | when(metaRS.getColumnName(22)).thenReturn("BINARY") 49 | when(metaRS.getColumnName(23)).thenReturn("BLOB") 50 | when(metaRS.getColumnName(24)).thenReturn("VARBINARY") 51 | when(metaRS.getColumnName(25)).thenReturn("LONGVARBINARY") 52 | when(metaRS.getColumnName(26)).thenReturn("DATE") 53 | when(metaRS.getColumnName(27)).thenReturn("TIME") 54 | when(metaRS.getColumnName(28)).thenReturn("TIMESTAMP") 55 | 56 | when(metaRS.getColumnLabel(1)).thenReturn("BOOLEAN") 57 | when(metaRS.getColumnLabel(2)).thenReturn("BIT") 58 | when(metaRS.getColumnLabel(3)).thenReturn("TINYINT") 59 | when(metaRS.getColumnLabel(4)).thenReturn("SMALLINT") 60 | when(metaRS.getColumnLabel(5)).thenReturn("INTEGER") 61 | when(metaRS.getColumnLabel(6)).thenReturn("BIGINT") 62 | when(metaRS.getColumnLabel(7)).thenReturn("REAL") 63 | when(metaRS.getColumnLabel(8)).thenReturn("FLOAT") 64 | when(metaRS.getColumnLabel(9)).thenReturn("DOUBLE") 65 | when(metaRS.getColumnLabel(10)).thenReturn("NUMERIC") 66 | when(metaRS.getColumnLabel(11)).thenReturn("DECIMAL") 67 | when(metaRS.getColumnLabel(12)).thenReturn("CHAR") 68 | when(metaRS.getColumnLabel(13)).thenReturn("VARCHAR") 69 | when(metaRS.getColumnLabel(14)).thenReturn("LONGVARCHAR") 70 | when(metaRS.getColumnLabel(15)).thenReturn("NCHAR") 71 | when(metaRS.getColumnLabel(16)).thenReturn("NVARCHAR") 72 | when(metaRS.getColumnLabel(17)).thenReturn("LONGNVARCHAR") 73 | when(metaRS.getColumnLabel(18)).thenReturn("CLOB") 74 | when(metaRS.getColumnLabel(19)).thenReturn("NCLOB") 75 | when(metaRS.getColumnLabel(20)).thenReturn("DATALINK") 76 | when(metaRS.getColumnLabel(21)).thenReturn("SQLXML") 77 | when(metaRS.getColumnLabel(22)).thenReturn("BINARY") 78 | when(metaRS.getColumnLabel(23)).thenReturn("BLOB") 79 | when(metaRS.getColumnLabel(24)).thenReturn("VARBINARY") 80 | when(metaRS.getColumnLabel(25)).thenReturn("LONGVARBINARY") 81 | when(metaRS.getColumnLabel(26)).thenReturn("DATE") 82 | when(metaRS.getColumnLabel(27)).thenReturn("TIME") 83 | when(metaRS.getColumnLabel(28)).thenReturn("TIMESTAMP") 84 | 85 | when(metaRS.getColumnType(1)).thenReturn(Types.BOOLEAN) 86 | when(metaRS.getColumnType(2)).thenReturn(Types.BIT) 87 | when(metaRS.getColumnType(3)).thenReturn(Types.TINYINT) 88 | when(metaRS.getColumnType(4)).thenReturn(Types.SMALLINT) 89 | when(metaRS.getColumnType(5)).thenReturn(Types.INTEGER) 90 | when(metaRS.getColumnType(6)).thenReturn(Types.BIGINT) 91 | when(metaRS.getColumnType(7)).thenReturn(Types.REAL) 92 | when(metaRS.getColumnType(8)).thenReturn(Types.FLOAT) 93 | when(metaRS.getColumnType(9)).thenReturn(Types.DOUBLE) 94 | when(metaRS.getColumnType(10)).thenReturn(Types.NUMERIC) 95 | when(metaRS.getColumnType(11)).thenReturn(Types.DECIMAL) 96 | when(metaRS.getColumnType(12)).thenReturn(Types.CHAR) 97 | when(metaRS.getColumnType(13)).thenReturn(Types.VARCHAR) 98 | when(metaRS.getColumnType(14)).thenReturn(Types.LONGVARCHAR) 99 | when(metaRS.getColumnType(15)).thenReturn(Types.NCHAR) 100 | when(metaRS.getColumnType(16)).thenReturn(Types.NVARCHAR) 101 | when(metaRS.getColumnType(17)).thenReturn(Types.LONGNVARCHAR) 102 | when(metaRS.getColumnType(18)).thenReturn(Types.CLOB) 103 | when(metaRS.getColumnType(19)).thenReturn(Types.NCLOB) 104 | when(metaRS.getColumnType(20)).thenReturn(Types.DATALINK) 105 | when(metaRS.getColumnType(21)).thenReturn(Types.SQLXML) 106 | when(metaRS.getColumnType(22)).thenReturn(Types.BINARY) 107 | when(metaRS.getColumnType(23)).thenReturn(Types.BLOB) 108 | when(metaRS.getColumnType(24)).thenReturn(Types.VARBINARY) 109 | when(metaRS.getColumnType(25)).thenReturn(Types.LONGVARBINARY) 110 | when(metaRS.getColumnType(26)).thenReturn(Types.DATE) 111 | when(metaRS.getColumnType(27)).thenReturn(Types.TIME) 112 | when(metaRS.getColumnType(28)).thenReturn(Types.TIMESTAMP) 113 | 114 | when(metaRS.isNullable(1)).thenReturn(ResultSetMetaData.columnNoNulls) 115 | when(metaRS.isNullable(2)).thenReturn(ResultSetMetaData.columnNoNulls) 116 | when(metaRS.isNullable(3)).thenReturn(ResultSetMetaData.columnNoNulls) 117 | when(metaRS.isNullable(4)).thenReturn(ResultSetMetaData.columnNoNulls) 118 | when(metaRS.isNullable(5)).thenReturn(ResultSetMetaData.columnNoNulls) 119 | when(metaRS.isNullable(6)).thenReturn(ResultSetMetaData.columnNoNulls) 120 | when(metaRS.isNullable(7)).thenReturn(ResultSetMetaData.columnNoNulls) 121 | when(metaRS.isNullable(8)).thenReturn(ResultSetMetaData.columnNoNulls) 122 | when(metaRS.isNullable(9)).thenReturn(ResultSetMetaData.columnNoNulls) 123 | when(metaRS.isNullable(10)).thenReturn(ResultSetMetaData.columnNoNulls) 124 | when(metaRS.isNullable(11)).thenReturn(ResultSetMetaData.columnNoNulls) 125 | when(metaRS.isNullable(12)).thenReturn(ResultSetMetaData.columnNoNulls) 126 | when(metaRS.isNullable(13)).thenReturn(ResultSetMetaData.columnNoNulls) 127 | when(metaRS.isNullable(14)).thenReturn(ResultSetMetaData.columnNoNulls) 128 | when(metaRS.isNullable(15)).thenReturn(ResultSetMetaData.columnNoNulls) 129 | when(metaRS.isNullable(16)).thenReturn(ResultSetMetaData.columnNoNulls) 130 | when(metaRS.isNullable(17)).thenReturn(ResultSetMetaData.columnNoNulls) 131 | when(metaRS.isNullable(18)).thenReturn(ResultSetMetaData.columnNoNulls) 132 | when(metaRS.isNullable(19)).thenReturn(ResultSetMetaData.columnNoNulls) 133 | when(metaRS.isNullable(20)).thenReturn(ResultSetMetaData.columnNoNulls) 134 | when(metaRS.isNullable(21)).thenReturn(ResultSetMetaData.columnNoNulls) 135 | when(metaRS.isNullable(22)).thenReturn(ResultSetMetaData.columnNoNulls) 136 | when(metaRS.isNullable(23)).thenReturn(ResultSetMetaData.columnNoNulls) 137 | when(metaRS.isNullable(24)).thenReturn(ResultSetMetaData.columnNoNulls) 138 | when(metaRS.isNullable(25)).thenReturn(ResultSetMetaData.columnNoNulls) 139 | when(metaRS.isNullable(26)).thenReturn(ResultSetMetaData.columnNoNulls) 140 | when(metaRS.isNullable(27)).thenReturn(ResultSetMetaData.columnNoNulls) 141 | when(metaRS.isNullable(28)).thenReturn(ResultSetMetaData.columnNoNulls) 142 | 143 | when(metaRS.getScale(10)).thenReturn(2) 144 | when(metaRS.getScale(11)).thenReturn(3) 145 | 146 | val s = dataConverter.convertSchema("test-procedure", metaRS) 147 | 148 | s.map(_.name) shouldBe Success("test-procedure") 149 | s.map(_.field("BOOLEAN").schema()) shouldBe Success(Schema.BOOLEAN_SCHEMA) 150 | s.map(_.field("BIT").schema()) shouldBe Success(Schema.INT8_SCHEMA) 151 | s.map(_.field("TINYINT").schema()) shouldBe Success(Schema.INT8_SCHEMA) 152 | s.map(_.field("SMALLINT").schema()) shouldBe Success(Schema.INT16_SCHEMA) 153 | s.map(_.field("INTEGER").schema()) shouldBe Success(Schema.INT32_SCHEMA) 154 | s.map(_.field("BIGINT").schema()) shouldBe Success(Schema.INT64_SCHEMA) 155 | s.map(_.field("REAL").schema()) shouldBe Success(Schema.FLOAT32_SCHEMA) 156 | s.map(_.field("FLOAT").schema()) shouldBe Success(Schema.FLOAT64_SCHEMA) 157 | s.map(_.field("DOUBLE").schema()) shouldBe Success(Schema.FLOAT64_SCHEMA) 158 | s.map(_.field("NUMERIC").schema()) shouldBe Success(Decimal.builder(2).build()) 159 | s.map(_.field("DECIMAL").schema()) shouldBe Success(Decimal.builder(3).build()) 160 | s.map(_.field("CHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA) 161 | s.map(_.field("VARCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA) 162 | s.map(_.field("LONGVARCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA) 163 | s.map(_.field("NCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA) 164 | s.map(_.field("NVARCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA) 165 | s.map(_.field("LONGNVARCHAR").schema()) shouldBe Success(Schema.STRING_SCHEMA) 166 | s.map(_.field("CLOB").schema()) shouldBe Success(Schema.STRING_SCHEMA) 167 | s.map(_.field("NCLOB").schema()) shouldBe Success(Schema.STRING_SCHEMA) 168 | s.map(_.field("DATALINK").schema()) shouldBe Success(Schema.STRING_SCHEMA) 169 | s.map(_.field("SQLXML").schema()) shouldBe Success(Schema.STRING_SCHEMA) 170 | s.map(_.field("BINARY").schema()) shouldBe Success(Schema.BYTES_SCHEMA) 171 | s.map(_.field("BLOB").schema()) shouldBe Success(Schema.BYTES_SCHEMA) 172 | s.map(_.field("VARBINARY").schema()) shouldBe Success(Schema.BYTES_SCHEMA) 173 | s.map(_.field("LONGVARBINARY").schema()) shouldBe Success(Schema.BYTES_SCHEMA) 174 | s.map(_.field("DATE").schema()) shouldBe Success(Date.builder().build()) 175 | s.map(_.field("TIME").schema()) shouldBe Success(Time.builder().build()) 176 | s.map(_.field("TIMESTAMP").schema()) shouldBe Success(Timestamp.builder().build()) 177 | } 178 | 179 | "create schema from result set metadata with optional columns" in { 180 | val metaRS = mock[ResultSetMetaData] 181 | 182 | when(metaRS.getColumnCount).thenReturn(28) 183 | 184 | when(metaRS.getColumnName(1)).thenReturn("BOOLEAN") 185 | when(metaRS.getColumnName(2)).thenReturn("BIT") 186 | when(metaRS.getColumnName(3)).thenReturn("TINYINT") 187 | when(metaRS.getColumnName(4)).thenReturn("SMALLINT") 188 | when(metaRS.getColumnName(5)).thenReturn("INTEGER") 189 | when(metaRS.getColumnName(6)).thenReturn("BIGINT") 190 | when(metaRS.getColumnName(7)).thenReturn("REAL") 191 | when(metaRS.getColumnName(8)).thenReturn("FLOAT") 192 | when(metaRS.getColumnName(9)).thenReturn("DOUBLE") 193 | when(metaRS.getColumnName(10)).thenReturn("NUMERIC") 194 | when(metaRS.getColumnName(11)).thenReturn("DECIMAL") 195 | when(metaRS.getColumnName(12)).thenReturn("CHAR") 196 | when(metaRS.getColumnName(13)).thenReturn("VARCHAR") 197 | when(metaRS.getColumnName(14)).thenReturn("LONGVARCHAR") 198 | when(metaRS.getColumnName(15)).thenReturn("NCHAR") 199 | when(metaRS.getColumnName(16)).thenReturn("NVARCHAR") 200 | when(metaRS.getColumnName(17)).thenReturn("LONGNVARCHAR") 201 | when(metaRS.getColumnName(18)).thenReturn("CLOB") 202 | when(metaRS.getColumnName(19)).thenReturn("NCLOB") 203 | when(metaRS.getColumnName(20)).thenReturn("DATALINK") 204 | when(metaRS.getColumnName(21)).thenReturn("SQLXML") 205 | when(metaRS.getColumnName(22)).thenReturn("BINARY") 206 | when(metaRS.getColumnName(23)).thenReturn("BLOB") 207 | when(metaRS.getColumnName(24)).thenReturn("VARBINARY") 208 | when(metaRS.getColumnName(25)).thenReturn("LONGVARBINARY") 209 | when(metaRS.getColumnName(26)).thenReturn("DATE") 210 | when(metaRS.getColumnName(27)).thenReturn("TIME") 211 | when(metaRS.getColumnName(28)).thenReturn("TIMESTAMP") 212 | 213 | when(metaRS.getColumnLabel(1)).thenReturn("BOOLEAN") 214 | when(metaRS.getColumnLabel(2)).thenReturn("BIT") 215 | when(metaRS.getColumnLabel(3)).thenReturn("TINYINT") 216 | when(metaRS.getColumnLabel(4)).thenReturn("SMALLINT") 217 | when(metaRS.getColumnLabel(5)).thenReturn("INTEGER") 218 | when(metaRS.getColumnLabel(6)).thenReturn("BIGINT") 219 | when(metaRS.getColumnLabel(7)).thenReturn("REAL") 220 | when(metaRS.getColumnLabel(8)).thenReturn("FLOAT") 221 | when(metaRS.getColumnLabel(9)).thenReturn("DOUBLE") 222 | when(metaRS.getColumnLabel(10)).thenReturn("NUMERIC") 223 | when(metaRS.getColumnLabel(11)).thenReturn("DECIMAL") 224 | when(metaRS.getColumnLabel(12)).thenReturn("CHAR") 225 | when(metaRS.getColumnLabel(13)).thenReturn("VARCHAR") 226 | when(metaRS.getColumnLabel(14)).thenReturn("LONGVARCHAR") 227 | when(metaRS.getColumnLabel(15)).thenReturn("NCHAR") 228 | when(metaRS.getColumnLabel(16)).thenReturn("NVARCHAR") 229 | when(metaRS.getColumnLabel(17)).thenReturn("LONGNVARCHAR") 230 | when(metaRS.getColumnLabel(18)).thenReturn("CLOB") 231 | when(metaRS.getColumnLabel(19)).thenReturn("NCLOB") 232 | when(metaRS.getColumnLabel(20)).thenReturn("DATALINK") 233 | when(metaRS.getColumnLabel(21)).thenReturn("SQLXML") 234 | when(metaRS.getColumnLabel(22)).thenReturn("BINARY") 235 | when(metaRS.getColumnLabel(23)).thenReturn("BLOB") 236 | when(metaRS.getColumnLabel(24)).thenReturn("VARBINARY") 237 | when(metaRS.getColumnLabel(25)).thenReturn("LONGVARBINARY") 238 | when(metaRS.getColumnLabel(26)).thenReturn("DATE") 239 | when(metaRS.getColumnLabel(27)).thenReturn("TIME") 240 | when(metaRS.getColumnLabel(28)).thenReturn("TIMESTAMP") 241 | 242 | when(metaRS.getColumnType(1)).thenReturn(Types.BOOLEAN) 243 | when(metaRS.getColumnType(2)).thenReturn(Types.BIT) 244 | when(metaRS.getColumnType(3)).thenReturn(Types.TINYINT) 245 | when(metaRS.getColumnType(4)).thenReturn(Types.SMALLINT) 246 | when(metaRS.getColumnType(5)).thenReturn(Types.INTEGER) 247 | when(metaRS.getColumnType(6)).thenReturn(Types.BIGINT) 248 | when(metaRS.getColumnType(7)).thenReturn(Types.REAL) 249 | when(metaRS.getColumnType(8)).thenReturn(Types.FLOAT) 250 | when(metaRS.getColumnType(9)).thenReturn(Types.DOUBLE) 251 | when(metaRS.getColumnType(10)).thenReturn(Types.NUMERIC) 252 | when(metaRS.getColumnType(11)).thenReturn(Types.DECIMAL) 253 | when(metaRS.getColumnType(12)).thenReturn(Types.CHAR) 254 | when(metaRS.getColumnType(13)).thenReturn(Types.VARCHAR) 255 | when(metaRS.getColumnType(14)).thenReturn(Types.LONGVARCHAR) 256 | when(metaRS.getColumnType(15)).thenReturn(Types.NCHAR) 257 | when(metaRS.getColumnType(16)).thenReturn(Types.NVARCHAR) 258 | when(metaRS.getColumnType(17)).thenReturn(Types.LONGNVARCHAR) 259 | when(metaRS.getColumnType(18)).thenReturn(Types.CLOB) 260 | when(metaRS.getColumnType(19)).thenReturn(Types.NCLOB) 261 | when(metaRS.getColumnType(20)).thenReturn(Types.DATALINK) 262 | when(metaRS.getColumnType(21)).thenReturn(Types.SQLXML) 263 | when(metaRS.getColumnType(22)).thenReturn(Types.BINARY) 264 | when(metaRS.getColumnType(23)).thenReturn(Types.BLOB) 265 | when(metaRS.getColumnType(24)).thenReturn(Types.VARBINARY) 266 | when(metaRS.getColumnType(25)).thenReturn(Types.LONGVARBINARY) 267 | when(metaRS.getColumnType(26)).thenReturn(Types.DATE) 268 | when(metaRS.getColumnType(27)).thenReturn(Types.TIME) 269 | when(metaRS.getColumnType(28)).thenReturn(Types.TIMESTAMP) 270 | 271 | when(metaRS.isNullable(1)).thenReturn(ResultSetMetaData.columnNullable) 272 | when(metaRS.isNullable(2)).thenReturn(ResultSetMetaData.columnNullable) 273 | when(metaRS.isNullable(3)).thenReturn(ResultSetMetaData.columnNullable) 274 | when(metaRS.isNullable(4)).thenReturn(ResultSetMetaData.columnNullable) 275 | when(metaRS.isNullable(5)).thenReturn(ResultSetMetaData.columnNullable) 276 | when(metaRS.isNullable(6)).thenReturn(ResultSetMetaData.columnNullable) 277 | when(metaRS.isNullable(7)).thenReturn(ResultSetMetaData.columnNullable) 278 | when(metaRS.isNullable(8)).thenReturn(ResultSetMetaData.columnNullable) 279 | when(metaRS.isNullable(9)).thenReturn(ResultSetMetaData.columnNullable) 280 | when(metaRS.isNullable(10)).thenReturn(ResultSetMetaData.columnNullable) 281 | when(metaRS.isNullable(11)).thenReturn(ResultSetMetaData.columnNullable) 282 | when(metaRS.isNullable(12)).thenReturn(ResultSetMetaData.columnNullable) 283 | when(metaRS.isNullable(13)).thenReturn(ResultSetMetaData.columnNullable) 284 | when(metaRS.isNullable(14)).thenReturn(ResultSetMetaData.columnNullable) 285 | when(metaRS.isNullable(15)).thenReturn(ResultSetMetaData.columnNullable) 286 | when(metaRS.isNullable(16)).thenReturn(ResultSetMetaData.columnNullable) 287 | when(metaRS.isNullable(17)).thenReturn(ResultSetMetaData.columnNullable) 288 | when(metaRS.isNullable(18)).thenReturn(ResultSetMetaData.columnNullable) 289 | when(metaRS.isNullable(19)).thenReturn(ResultSetMetaData.columnNullable) 290 | when(metaRS.isNullable(20)).thenReturn(ResultSetMetaData.columnNullable) 291 | when(metaRS.isNullable(21)).thenReturn(ResultSetMetaData.columnNullable) 292 | when(metaRS.isNullable(22)).thenReturn(ResultSetMetaData.columnNullable) 293 | when(metaRS.isNullable(23)).thenReturn(ResultSetMetaData.columnNullable) 294 | when(metaRS.isNullable(24)).thenReturn(ResultSetMetaData.columnNullable) 295 | when(metaRS.isNullable(25)).thenReturn(ResultSetMetaData.columnNullable) 296 | when(metaRS.isNullable(26)).thenReturn(ResultSetMetaData.columnNullable) 297 | when(metaRS.isNullable(27)).thenReturn(ResultSetMetaData.columnNullable) 298 | when(metaRS.isNullable(28)).thenReturn(ResultSetMetaData.columnNullable) 299 | 300 | when(metaRS.getScale(10)).thenReturn(2) 301 | when(metaRS.getScale(11)).thenReturn(3) 302 | 303 | val s = dataConverter.convertSchema("test-procedure", metaRS) 304 | 305 | s.map(_.name) shouldBe Success("test-procedure") 306 | s.map(_.field("BOOLEAN").schema()) shouldBe Success(Schema.OPTIONAL_BOOLEAN_SCHEMA) 307 | s.map(_.field("BIT").schema()) shouldBe Success(Schema.OPTIONAL_INT8_SCHEMA) 308 | s.map(_.field("TINYINT").schema()) shouldBe Success(Schema.OPTIONAL_INT8_SCHEMA) 309 | s.map(_.field("SMALLINT").schema()) shouldBe Success(Schema.OPTIONAL_INT16_SCHEMA) 310 | s.map(_.field("INTEGER").schema()) shouldBe Success(Schema.OPTIONAL_INT32_SCHEMA) 311 | s.map(_.field("BIGINT").schema()) shouldBe Success(Schema.OPTIONAL_INT64_SCHEMA) 312 | s.map(_.field("REAL").schema()) shouldBe Success(Schema.OPTIONAL_FLOAT32_SCHEMA) 313 | s.map(_.field("FLOAT").schema()) shouldBe Success(Schema.OPTIONAL_FLOAT64_SCHEMA) 314 | s.map(_.field("DOUBLE").schema()) shouldBe Success(Schema.OPTIONAL_FLOAT64_SCHEMA) 315 | s.map(_.field("NUMERIC").schema()) shouldBe Success(Decimal.builder(2).optional().build()) 316 | s.map(_.field("DECIMAL").schema()) shouldBe Success(Decimal.builder(3).optional().build()) 317 | s.map(_.field("CHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 318 | s.map(_.field("VARCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 319 | s.map(_.field("LONGVARCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 320 | s.map(_.field("NCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 321 | s.map(_.field("NVARCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 322 | s.map(_.field("LONGNVARCHAR").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 323 | s.map(_.field("CLOB").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 324 | s.map(_.field("NCLOB").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 325 | s.map(_.field("DATALINK").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 326 | s.map(_.field("SQLXML").schema()) shouldBe Success(Schema.OPTIONAL_STRING_SCHEMA) 327 | s.map(_.field("BINARY").schema()) shouldBe Success(Schema.OPTIONAL_BYTES_SCHEMA) 328 | s.map(_.field("BLOB").schema()) shouldBe Success(Schema.OPTIONAL_BYTES_SCHEMA) 329 | s.map(_.field("VARBINARY").schema()) shouldBe Success(Schema.OPTIONAL_BYTES_SCHEMA) 330 | s.map(_.field("LONGVARBINARY").schema()) shouldBe Success(Schema.OPTIONAL_BYTES_SCHEMA) 331 | s.map(_.field("DATE").schema()) shouldBe Success(Date.builder().optional().build()) 332 | s.map(_.field("TIME").schema()) shouldBe Success(Time.builder().optional().build()) 333 | s.map(_.field("TIMESTAMP").schema()) shouldBe Success(Timestamp.builder().optional().build()) 334 | } 335 | 336 | "skip unsupported SQL types while creating schema" in { 337 | val metaRS = mock[ResultSetMetaData] 338 | 339 | when(metaRS.getColumnCount).thenReturn(2) 340 | when(metaRS.getColumnName(1)).thenReturn("KEY") 341 | when(metaRS.getColumnName(2)).thenReturn("VALUE") 342 | when(metaRS.getColumnLabel(1)).thenReturn(null) 343 | when(metaRS.getColumnLabel(2)).thenReturn("") 344 | when(metaRS.getColumnType(1)).thenReturn(Types.INTEGER) 345 | when(metaRS.getColumnType(2)).thenReturn(Types.JAVA_OBJECT) 346 | when(metaRS.isNullable(1)).thenReturn(ResultSetMetaData.columnNoNulls) 347 | when(metaRS.isNullable(2)).thenReturn(ResultSetMetaData.columnNullableUnknown) 348 | 349 | val s = dataConverter.convertSchema("test-procedure-unsupported", metaRS) 350 | 351 | s.map(_.name) shouldBe Success("test-procedure-unsupported") 352 | s.map(_.fields.size) shouldBe Success(1) 353 | s.map(_.field("KEY").schema()) shouldBe Success(Schema.INT32_SCHEMA) 354 | } 355 | 356 | "convert record from result set with non optional columns" in { 357 | val rS = mock[ResultSet] 358 | val metaRS = mock[ResultSetMetaData] 359 | val blob = mock[Blob] 360 | val clob = mock[Clob] 361 | val nClob = mock[NClob] 362 | val sqlXml = mock[SQLXML] 363 | 364 | val builder = SchemaBuilder.struct().name("test") 365 | builder.field("BOOLEAN", Schema.BOOLEAN_SCHEMA) 366 | builder.field("BIT", Schema.INT8_SCHEMA) 367 | builder.field("TINYINT", Schema.INT8_SCHEMA) 368 | builder.field("SMALLINT", Schema.INT16_SCHEMA) 369 | builder.field("INTEGER", Schema.INT32_SCHEMA) 370 | builder.field("BIGINT", Schema.INT64_SCHEMA) 371 | builder.field("REAL", Schema.FLOAT32_SCHEMA) 372 | builder.field("FLOAT", Schema.FLOAT64_SCHEMA) 373 | builder.field("DOUBLE", Schema.FLOAT64_SCHEMA) 374 | builder.field("NUMERIC", Decimal.builder(1).build()) 375 | builder.field("DECIMAL", Decimal.builder(1).build()) 376 | builder.field("CHAR", Schema.STRING_SCHEMA) 377 | builder.field("VARCHAR", Schema.STRING_SCHEMA) 378 | builder.field("LONGVARCHAR", Schema.STRING_SCHEMA) 379 | builder.field("NCHAR", Schema.STRING_SCHEMA) 380 | builder.field("NVARCHAR", Schema.STRING_SCHEMA) 381 | builder.field("LONGNVARCHAR", Schema.STRING_SCHEMA) 382 | builder.field("CLOB", Schema.STRING_SCHEMA) 383 | builder.field("NCLOB", Schema.STRING_SCHEMA) 384 | builder.field("DATALINK", Schema.STRING_SCHEMA) 385 | builder.field("SQLXML", Schema.STRING_SCHEMA) 386 | builder.field("BINARY", Schema.BYTES_SCHEMA) 387 | builder.field("VARBINARY", Schema.BYTES_SCHEMA) 388 | builder.field("LONGVARBINARY", Schema.BYTES_SCHEMA) 389 | builder.field("BLOB", Schema.BYTES_SCHEMA) 390 | builder.field("DATE", Date.builder().build()) 391 | builder.field("TIME", Time.builder().build()) 392 | builder.field("TIMESTAMP", Timestamp.builder().build()) 393 | val schema = builder.build() 394 | 395 | when(rS.wasNull()).thenReturn(false) 396 | when(rS.getMetaData).thenReturn(metaRS) 397 | when(metaRS.getColumnCount).thenReturn(28) 398 | 399 | when(metaRS.getColumnName(1)).thenReturn("BOOLEAN") 400 | when(metaRS.getColumnName(2)).thenReturn("BIT") 401 | when(metaRS.getColumnName(3)).thenReturn("TINYINT") 402 | when(metaRS.getColumnName(4)).thenReturn("SMALLINT") 403 | when(metaRS.getColumnName(5)).thenReturn("INTEGER") 404 | when(metaRS.getColumnName(6)).thenReturn("BIGINT") 405 | when(metaRS.getColumnName(7)).thenReturn("REAL") 406 | when(metaRS.getColumnName(8)).thenReturn("FLOAT") 407 | when(metaRS.getColumnName(9)).thenReturn("DOUBLE") 408 | when(metaRS.getColumnName(10)).thenReturn("NUMERIC") 409 | when(metaRS.getColumnName(11)).thenReturn("DECIMAL") 410 | when(metaRS.getColumnName(12)).thenReturn("CHAR") 411 | when(metaRS.getColumnName(13)).thenReturn("VARCHAR") 412 | when(metaRS.getColumnName(14)).thenReturn("LONGVARCHAR") 413 | when(metaRS.getColumnName(15)).thenReturn("NCHAR") 414 | when(metaRS.getColumnName(16)).thenReturn("NVARCHAR") 415 | when(metaRS.getColumnName(17)).thenReturn("LONGNVARCHAR") 416 | when(metaRS.getColumnName(18)).thenReturn("CLOB") 417 | when(metaRS.getColumnName(19)).thenReturn("NCLOB") 418 | when(metaRS.getColumnName(20)).thenReturn("DATALINK") 419 | when(metaRS.getColumnName(21)).thenReturn("SQLXML") 420 | when(metaRS.getColumnName(22)).thenReturn("BINARY") 421 | when(metaRS.getColumnName(23)).thenReturn("VARBINARY") 422 | when(metaRS.getColumnName(24)).thenReturn("LONGVARBINARY") 423 | when(metaRS.getColumnName(25)).thenReturn("BLOB") 424 | when(metaRS.getColumnName(26)).thenReturn("DATE") 425 | when(metaRS.getColumnName(27)).thenReturn("TIME") 426 | when(metaRS.getColumnName(28)).thenReturn("TIMESTAMP") 427 | 428 | when(metaRS.getColumnLabel(1)).thenReturn("BOOLEAN") 429 | when(metaRS.getColumnLabel(2)).thenReturn("BIT") 430 | when(metaRS.getColumnLabel(3)).thenReturn("TINYINT") 431 | when(metaRS.getColumnLabel(4)).thenReturn("SMALLINT") 432 | when(metaRS.getColumnLabel(5)).thenReturn("INTEGER") 433 | when(metaRS.getColumnLabel(6)).thenReturn("BIGINT") 434 | when(metaRS.getColumnLabel(7)).thenReturn("REAL") 435 | when(metaRS.getColumnLabel(8)).thenReturn("FLOAT") 436 | when(metaRS.getColumnLabel(9)).thenReturn("DOUBLE") 437 | when(metaRS.getColumnLabel(10)).thenReturn("NUMERIC") 438 | when(metaRS.getColumnLabel(11)).thenReturn("DECIMAL") 439 | when(metaRS.getColumnLabel(12)).thenReturn("CHAR") 440 | when(metaRS.getColumnLabel(13)).thenReturn("VARCHAR") 441 | when(metaRS.getColumnLabel(14)).thenReturn("LONGVARCHAR") 442 | when(metaRS.getColumnLabel(15)).thenReturn("NCHAR") 443 | when(metaRS.getColumnLabel(16)).thenReturn("NVARCHAR") 444 | when(metaRS.getColumnLabel(17)).thenReturn("LONGNVARCHAR") 445 | when(metaRS.getColumnLabel(18)).thenReturn("CLOB") 446 | when(metaRS.getColumnLabel(19)).thenReturn("NCLOB") 447 | when(metaRS.getColumnLabel(20)).thenReturn("DATALINK") 448 | when(metaRS.getColumnLabel(21)).thenReturn("SQLXML") 449 | when(metaRS.getColumnLabel(22)).thenReturn("BINARY") 450 | when(metaRS.getColumnLabel(23)).thenReturn("VARBINARY") 451 | when(metaRS.getColumnLabel(24)).thenReturn("LONGVARBINARY") 452 | when(metaRS.getColumnLabel(25)).thenReturn("BLOB") 453 | when(metaRS.getColumnLabel(26)).thenReturn("DATE") 454 | when(metaRS.getColumnLabel(27)).thenReturn("TIME") 455 | when(metaRS.getColumnLabel(28)).thenReturn("TIMESTAMP") 456 | 457 | when(metaRS.getColumnType(1)).thenReturn(Types.BOOLEAN) 458 | when(metaRS.getColumnType(2)).thenReturn(Types.BIT) 459 | when(metaRS.getColumnType(3)).thenReturn(Types.TINYINT) 460 | when(metaRS.getColumnType(4)).thenReturn(Types.SMALLINT) 461 | when(metaRS.getColumnType(5)).thenReturn(Types.INTEGER) 462 | when(metaRS.getColumnType(6)).thenReturn(Types.BIGINT) 463 | when(metaRS.getColumnType(7)).thenReturn(Types.REAL) 464 | when(metaRS.getColumnType(8)).thenReturn(Types.FLOAT) 465 | when(metaRS.getColumnType(9)).thenReturn(Types.DOUBLE) 466 | when(metaRS.getColumnType(10)).thenReturn(Types.NUMERIC) 467 | when(metaRS.getColumnType(11)).thenReturn(Types.DECIMAL) 468 | when(metaRS.getColumnType(12)).thenReturn(Types.CHAR) 469 | when(metaRS.getColumnType(13)).thenReturn(Types.VARCHAR) 470 | when(metaRS.getColumnType(14)).thenReturn(Types.LONGVARCHAR) 471 | when(metaRS.getColumnType(15)).thenReturn(Types.NCHAR) 472 | when(metaRS.getColumnType(16)).thenReturn(Types.NVARCHAR) 473 | when(metaRS.getColumnType(17)).thenReturn(Types.LONGNVARCHAR) 474 | when(metaRS.getColumnType(18)).thenReturn(Types.CLOB) 475 | when(metaRS.getColumnType(19)).thenReturn(Types.NCLOB) 476 | when(metaRS.getColumnType(20)).thenReturn(Types.DATALINK) 477 | when(metaRS.getColumnType(21)).thenReturn(Types.SQLXML) 478 | when(metaRS.getColumnType(22)).thenReturn(Types.BINARY) 479 | when(metaRS.getColumnType(23)).thenReturn(Types.VARBINARY) 480 | when(metaRS.getColumnType(24)).thenReturn(Types.LONGVARBINARY) 481 | when(metaRS.getColumnType(25)).thenReturn(Types.BLOB) 482 | when(metaRS.getColumnType(26)).thenReturn(Types.DATE) 483 | when(metaRS.getColumnType(27)).thenReturn(Types.TIME) 484 | when(metaRS.getColumnType(28)).thenReturn(Types.TIMESTAMP) 485 | 486 | when(blob.getBytes(1, blob.length.toInt)).thenReturn("25".toCharArray.map(_.toByte)) 487 | when(clob.getSubString(1, clob.length.toInt)).thenReturn("18") 488 | when(nClob.getSubString(1, clob.length.toInt)).thenReturn("19") 489 | when(sqlXml.getString).thenReturn("21") 490 | 491 | when(rS.getBoolean(1)).thenReturn(true) 492 | when(rS.getByte(2)).thenReturn(2.toByte) 493 | when(rS.getByte(3)).thenReturn(3.toByte) 494 | when(rS.getShort(4)).thenReturn(4.toShort) 495 | when(rS.getInt(5)).thenReturn(5) 496 | when(rS.getLong(6)).thenReturn(6L) 497 | when(rS.getFloat(7)).thenReturn(7.0F) 498 | when(rS.getDouble(8)).thenReturn(8.0) 499 | when(rS.getDouble(9)).thenReturn(9.0) 500 | when(rS.getBigDecimal(10)).thenReturn(new BigDecimal("10.0")) 501 | when(rS.getBigDecimal(11)).thenReturn(new BigDecimal("11.0")) 502 | when(rS.getString(12)).thenReturn("12") 503 | when(rS.getString(13)).thenReturn("13") 504 | when(rS.getString(14)).thenReturn("14") 505 | when(rS.getNString(15)).thenReturn("15") 506 | when(rS.getNString(16)).thenReturn("16") 507 | when(rS.getNString(17)).thenReturn("17") 508 | when(rS.getClob(18)).thenReturn(clob) 509 | when(rS.getNClob(19)).thenReturn(nClob) 510 | when(rS.getURL(20)).thenReturn(new URL("http://20")) 511 | when(rS.getSQLXML(21)).thenReturn(sqlXml) 512 | when(rS.getBytes(22)).thenReturn("22".toCharArray.map(_.toByte)) 513 | when(rS.getBytes(23)).thenReturn("23".toCharArray.map(_.toByte)) 514 | when(rS.getBytes(24)).thenReturn("24".toCharArray.map(_.toByte)) 515 | when(rS.getBlob(25)).thenReturn(blob) 516 | when(rS.getDate(same(26), any())).thenReturn(new SqlDate(26L)) 517 | when(rS.getTime(same(27), any())).thenReturn(new SqlTime(27L)) 518 | when(rS.getTimestamp(same(28), any())).thenReturn(new SqlTimestamp(28L)) 519 | 520 | val r = dataConverter.convertRecord(schema, rS) 521 | 522 | r.map(_.schema) shouldBe Success(schema) 523 | r.map(_.getBoolean("BOOLEAN")) shouldBe Success(true) 524 | r.map(_.getInt8("BIT")) shouldBe Success(2) 525 | r.map(_.getInt8("TINYINT")) shouldBe Success(3) 526 | r.map(_.getInt16("SMALLINT")) shouldBe Success(4) 527 | r.map(_.getInt32("INTEGER")) shouldBe Success(5) 528 | r.map(_.getInt64("BIGINT")) shouldBe Success(6L) 529 | r.map(_.getFloat32("REAL")) shouldBe Success(7.0F) 530 | r.map(_.getFloat64("FLOAT")) shouldBe Success(8.0F) 531 | r.map(_.getFloat64("DOUBLE")) shouldBe Success(9.0F) 532 | r.map(_.get("NUMERIC")) shouldBe Success(new BigDecimal("10.0")) 533 | r.map(_.get("DECIMAL")) shouldBe Success(new BigDecimal("11.0")) 534 | r.map(_.getString("CHAR")) shouldBe Success("12") 535 | r.map(_.getString("VARCHAR")) shouldBe Success("13") 536 | r.map(_.getString("LONGVARCHAR")) shouldBe Success("14") 537 | r.map(_.getString("NCHAR")) shouldBe Success("15") 538 | r.map(_.getString("NVARCHAR")) shouldBe Success("16") 539 | r.map(_.getString("LONGNVARCHAR")) shouldBe Success("17") 540 | r.map(_.getString("CLOB")) shouldBe Success("18") 541 | r.map(_.getString("NCLOB")) shouldBe Success("19") 542 | r.map(_.getString("DATALINK")) shouldBe Success("http://20") 543 | r.map(_.getString("SQLXML")) shouldBe Success("21") 544 | r.map(_.getBytes("BINARY").toSeq) shouldBe Success(Seq('2', '2')) 545 | r.map(_.getBytes("VARBINARY").toSeq) shouldBe Success(Seq('2', '3')) 546 | r.map(_.getBytes("LONGVARBINARY").toSeq) shouldBe Success(Seq('2', '4')) 547 | r.map(_.getBytes("BLOB").toSeq) shouldBe Success(Seq('2', '5')) 548 | r.map(_.get("DATE")) shouldBe Success(new SqlDate(26L)) 549 | r.map(_.get("TIME")) shouldBe Success(new SqlTime(27L)) 550 | r.map(_.get("TIMESTAMP")) shouldBe Success(new SqlTimestamp(28L)) 551 | } 552 | 553 | "convert record from result set with optional columns" in { 554 | val rS = mock[ResultSet] 555 | val metaRS = mock[ResultSetMetaData] 556 | 557 | val builder = SchemaBuilder.struct().name("test") 558 | builder.field("BOOLEAN", Schema.OPTIONAL_BOOLEAN_SCHEMA) 559 | builder.field("BIT", Schema.OPTIONAL_INT8_SCHEMA) 560 | builder.field("TINYINT", Schema.OPTIONAL_INT8_SCHEMA) 561 | builder.field("SMALLINT", Schema.OPTIONAL_INT16_SCHEMA) 562 | builder.field("INTEGER", Schema.OPTIONAL_INT32_SCHEMA) 563 | builder.field("BIGINT", Schema.OPTIONAL_INT64_SCHEMA) 564 | builder.field("REAL", Schema.OPTIONAL_FLOAT32_SCHEMA) 565 | builder.field("FLOAT", Schema.OPTIONAL_FLOAT64_SCHEMA) 566 | builder.field("DOUBLE", Schema.OPTIONAL_FLOAT64_SCHEMA) 567 | builder.field("NUMERIC", Decimal.builder(1).optional().build()) 568 | builder.field("DECIMAL", Decimal.builder(1).optional().build()) 569 | builder.field("CHAR", Schema.OPTIONAL_STRING_SCHEMA) 570 | builder.field("VARCHAR", Schema.OPTIONAL_STRING_SCHEMA) 571 | builder.field("LONGVARCHAR", Schema.OPTIONAL_STRING_SCHEMA) 572 | builder.field("NCHAR", Schema.OPTIONAL_STRING_SCHEMA) 573 | builder.field("NVARCHAR", Schema.OPTIONAL_STRING_SCHEMA) 574 | builder.field("LONGNVARCHAR", Schema.OPTIONAL_STRING_SCHEMA) 575 | builder.field("CLOB", Schema.OPTIONAL_STRING_SCHEMA) 576 | builder.field("NCLOB", Schema.OPTIONAL_STRING_SCHEMA) 577 | builder.field("DATALINK", Schema.OPTIONAL_STRING_SCHEMA) 578 | builder.field("SQLXML", Schema.OPTIONAL_STRING_SCHEMA) 579 | builder.field("BINARY", Schema.OPTIONAL_BYTES_SCHEMA) 580 | builder.field("VARBINARY", Schema.OPTIONAL_BYTES_SCHEMA) 581 | builder.field("LONGVARBINARY", Schema.OPTIONAL_BYTES_SCHEMA) 582 | builder.field("BLOB", Schema.OPTIONAL_BYTES_SCHEMA) 583 | builder.field("DATE", Date.builder().optional().build()) 584 | builder.field("TIME", Time.builder().optional().build()) 585 | builder.field("TIMESTAMP", Timestamp.builder().optional().build()) 586 | val schema = builder.build() 587 | 588 | when(rS.wasNull()).thenReturn(false) 589 | when(rS.getMetaData).thenReturn(metaRS) 590 | when(metaRS.getColumnCount).thenReturn(28) 591 | 592 | when(metaRS.getColumnName(1)).thenReturn("BOOLEAN") 593 | when(metaRS.getColumnName(2)).thenReturn("BIT") 594 | when(metaRS.getColumnName(3)).thenReturn("TINYINT") 595 | when(metaRS.getColumnName(4)).thenReturn("SMALLINT") 596 | when(metaRS.getColumnName(5)).thenReturn("INTEGER") 597 | when(metaRS.getColumnName(6)).thenReturn("BIGINT") 598 | when(metaRS.getColumnName(7)).thenReturn("REAL") 599 | when(metaRS.getColumnName(8)).thenReturn("FLOAT") 600 | when(metaRS.getColumnName(9)).thenReturn("DOUBLE") 601 | when(metaRS.getColumnName(10)).thenReturn("NUMERIC") 602 | when(metaRS.getColumnName(11)).thenReturn("DECIMAL") 603 | when(metaRS.getColumnName(12)).thenReturn("CHAR") 604 | when(metaRS.getColumnName(13)).thenReturn("VARCHAR") 605 | when(metaRS.getColumnName(14)).thenReturn("LONGVARCHAR") 606 | when(metaRS.getColumnName(15)).thenReturn("NCHAR") 607 | when(metaRS.getColumnName(16)).thenReturn("NVARCHAR") 608 | when(metaRS.getColumnName(17)).thenReturn("LONGNVARCHAR") 609 | when(metaRS.getColumnName(18)).thenReturn("CLOB") 610 | when(metaRS.getColumnName(19)).thenReturn("NCLOB") 611 | when(metaRS.getColumnName(20)).thenReturn("DATALINK") 612 | when(metaRS.getColumnName(21)).thenReturn("SQLXML") 613 | when(metaRS.getColumnName(22)).thenReturn("BINARY") 614 | when(metaRS.getColumnName(23)).thenReturn("VARBINARY") 615 | when(metaRS.getColumnName(24)).thenReturn("LONGVARBINARY") 616 | when(metaRS.getColumnName(25)).thenReturn("BLOB") 617 | when(metaRS.getColumnName(26)).thenReturn("DATE") 618 | when(metaRS.getColumnName(27)).thenReturn("TIME") 619 | when(metaRS.getColumnName(28)).thenReturn("TIMESTAMP") 620 | 621 | when(metaRS.getColumnLabel(1)).thenReturn("BOOLEAN") 622 | when(metaRS.getColumnLabel(2)).thenReturn("BIT") 623 | when(metaRS.getColumnLabel(3)).thenReturn("TINYINT") 624 | when(metaRS.getColumnLabel(4)).thenReturn("SMALLINT") 625 | when(metaRS.getColumnLabel(5)).thenReturn("INTEGER") 626 | when(metaRS.getColumnLabel(6)).thenReturn("BIGINT") 627 | when(metaRS.getColumnLabel(7)).thenReturn("REAL") 628 | when(metaRS.getColumnLabel(8)).thenReturn("FLOAT") 629 | when(metaRS.getColumnLabel(9)).thenReturn("DOUBLE") 630 | when(metaRS.getColumnLabel(10)).thenReturn("NUMERIC") 631 | when(metaRS.getColumnLabel(11)).thenReturn("DECIMAL") 632 | when(metaRS.getColumnLabel(12)).thenReturn("CHAR") 633 | when(metaRS.getColumnLabel(13)).thenReturn("VARCHAR") 634 | when(metaRS.getColumnLabel(14)).thenReturn("LONGVARCHAR") 635 | when(metaRS.getColumnLabel(15)).thenReturn("NCHAR") 636 | when(metaRS.getColumnLabel(16)).thenReturn("NVARCHAR") 637 | when(metaRS.getColumnLabel(17)).thenReturn("LONGNVARCHAR") 638 | when(metaRS.getColumnLabel(18)).thenReturn("CLOB") 639 | when(metaRS.getColumnLabel(19)).thenReturn("NCLOB") 640 | when(metaRS.getColumnLabel(20)).thenReturn("DATALINK") 641 | when(metaRS.getColumnLabel(21)).thenReturn("SQLXML") 642 | when(metaRS.getColumnLabel(22)).thenReturn("BINARY") 643 | when(metaRS.getColumnLabel(23)).thenReturn("VARBINARY") 644 | when(metaRS.getColumnLabel(24)).thenReturn("LONGVARBINARY") 645 | when(metaRS.getColumnLabel(25)).thenReturn("BLOB") 646 | when(metaRS.getColumnLabel(26)).thenReturn("DATE") 647 | when(metaRS.getColumnLabel(27)).thenReturn("TIME") 648 | when(metaRS.getColumnLabel(28)).thenReturn("TIMESTAMP") 649 | 650 | when(metaRS.getColumnType(1)).thenReturn(Types.BOOLEAN) 651 | when(metaRS.getColumnType(2)).thenReturn(Types.BIT) 652 | when(metaRS.getColumnType(3)).thenReturn(Types.TINYINT) 653 | when(metaRS.getColumnType(4)).thenReturn(Types.SMALLINT) 654 | when(metaRS.getColumnType(5)).thenReturn(Types.INTEGER) 655 | when(metaRS.getColumnType(6)).thenReturn(Types.BIGINT) 656 | when(metaRS.getColumnType(7)).thenReturn(Types.REAL) 657 | when(metaRS.getColumnType(8)).thenReturn(Types.FLOAT) 658 | when(metaRS.getColumnType(9)).thenReturn(Types.DOUBLE) 659 | when(metaRS.getColumnType(10)).thenReturn(Types.NUMERIC) 660 | when(metaRS.getColumnType(11)).thenReturn(Types.DECIMAL) 661 | when(metaRS.getColumnType(12)).thenReturn(Types.CHAR) 662 | when(metaRS.getColumnType(13)).thenReturn(Types.VARCHAR) 663 | when(metaRS.getColumnType(14)).thenReturn(Types.LONGVARCHAR) 664 | when(metaRS.getColumnType(15)).thenReturn(Types.NCHAR) 665 | when(metaRS.getColumnType(16)).thenReturn(Types.NVARCHAR) 666 | when(metaRS.getColumnType(17)).thenReturn(Types.LONGNVARCHAR) 667 | when(metaRS.getColumnType(18)).thenReturn(Types.CLOB) 668 | when(metaRS.getColumnType(19)).thenReturn(Types.NCLOB) 669 | when(metaRS.getColumnType(20)).thenReturn(Types.DATALINK) 670 | when(metaRS.getColumnType(21)).thenReturn(Types.SQLXML) 671 | when(metaRS.getColumnType(22)).thenReturn(Types.BINARY) 672 | when(metaRS.getColumnType(23)).thenReturn(Types.VARBINARY) 673 | when(metaRS.getColumnType(24)).thenReturn(Types.LONGVARBINARY) 674 | when(metaRS.getColumnType(25)).thenReturn(Types.BLOB) 675 | when(metaRS.getColumnType(26)).thenReturn(Types.DATE) 676 | when(metaRS.getColumnType(27)).thenReturn(Types.TIME) 677 | when(metaRS.getColumnType(28)).thenReturn(Types.TIMESTAMP) 678 | 679 | when(rS.getBigDecimal(10)).thenReturn(null) 680 | when(rS.getBigDecimal(11)).thenReturn(null) 681 | when(rS.getString(12)).thenReturn(null) 682 | when(rS.getString(13)).thenReturn(null) 683 | when(rS.getString(14)).thenReturn(null) 684 | when(rS.getNString(15)).thenReturn(null) 685 | when(rS.getNString(16)).thenReturn(null) 686 | when(rS.getNString(17)).thenReturn(null) 687 | when(rS.getClob(18)).thenReturn(null) 688 | when(rS.getNClob(19)).thenReturn(null) 689 | when(rS.getURL(20)).thenReturn(null) 690 | when(rS.getSQLXML(21)).thenReturn(null) 691 | when(rS.getBytes(22)).thenReturn(null) 692 | when(rS.getBytes(23)).thenReturn(null) 693 | when(rS.getBytes(24)).thenReturn(null) 694 | when(rS.getBlob(25)).thenReturn(null) 695 | when(rS.getDate(same(26), any())).thenReturn(null) 696 | when(rS.getTime(same(27), any())).thenReturn(null) 697 | when(rS.getTimestamp(same(28), any())).thenReturn(null) 698 | 699 | val r = dataConverter.convertRecord(schema, rS) 700 | 701 | r.map(_.schema) shouldBe Success(schema) 702 | r.map(_.getBoolean("BOOLEAN")) shouldBe Success(false) 703 | r.map(_.getInt8("BIT")) shouldBe Success(0) 704 | r.map(_.getInt8("TINYINT")) shouldBe Success(0) 705 | r.map(_.getInt16("SMALLINT")) shouldBe Success(0) 706 | r.map(_.getInt32("INTEGER")) shouldBe Success(0) 707 | r.map(_.getInt64("BIGINT")) shouldBe Success(0L) 708 | r.map(_.getFloat32("REAL")) shouldBe Success(0.0F) 709 | r.map(_.getFloat64("FLOAT")) shouldBe Success(0.0F) 710 | r.map(_.getFloat64("DOUBLE")) shouldBe Success(0.0F) 711 | r.map(_.get("NUMERIC")) shouldBe Success(null) 712 | r.map(_.get("DECIMAL")) shouldBe Success(null) 713 | r.map(_.getString("CHAR")) shouldBe Success(null) 714 | r.map(_.getString("VARCHAR")) shouldBe Success(null) 715 | r.map(_.getString("LONGVARCHAR")) shouldBe Success(null) 716 | r.map(_.getString("NCHAR")) shouldBe Success(null) 717 | r.map(_.getString("NVARCHAR")) shouldBe Success(null) 718 | r.map(_.getString("LONGNVARCHAR")) shouldBe Success(null) 719 | r.map(_.getString("CLOB")) shouldBe Success(null) 720 | r.map(_.getString("NCLOB")) shouldBe Success(null) 721 | r.map(_.getString("DATALINK")) shouldBe Success(null) 722 | r.map(_.getString("SQLXML")) shouldBe Success(null) 723 | r.map(_.getBytes("BINARY")) shouldBe Success(null) 724 | r.map(_.getBytes("VARBINARY")) shouldBe Success(null) 725 | r.map(_.getBytes("LONGVARBINARY")) shouldBe Success(null) 726 | r.map(_.getBytes("BLOB")) shouldBe Success(null) 727 | r.map(_.get("DATE")) shouldBe Success(null) 728 | r.map(_.get("TIME")) shouldBe Success(null) 729 | r.map(_.get("TIMESTAMP")) shouldBe Success(null) 730 | } 731 | 732 | "return null for unsupported SQL types while converting record" in { 733 | val rS = mock[ResultSet] 734 | val metaRS = mock[ResultSetMetaData] 735 | 736 | val builder = SchemaBuilder.struct().name("test") 737 | builder.field("KEY", Schema.INT32_SCHEMA) 738 | builder.field("VALUE", Schema.OPTIONAL_STRING_SCHEMA) 739 | val schema = builder.build() 740 | 741 | when(rS.getMetaData).thenReturn(metaRS) 742 | when(metaRS.getColumnCount).thenReturn(2) 743 | when(metaRS.getColumnName(1)).thenReturn("KEY") 744 | when(metaRS.getColumnName(2)).thenReturn("VALUE") 745 | when(metaRS.getColumnLabel(1)).thenReturn("KEY") 746 | when(metaRS.getColumnLabel(2)).thenReturn("VALUE") 747 | when(metaRS.getColumnType(1)).thenReturn(Types.INTEGER) 748 | when(metaRS.getColumnType(2)).thenReturn(Types.NULL) 749 | when(rS.getInt(1)).thenReturn(5) 750 | 751 | val r = dataConverter.convertRecord(schema, rS) 752 | 753 | r.map(_.schema) shouldBe Success(schema) 754 | r.map(_.getInt32("KEY")) shouldBe Success(5) 755 | r.map(_.getString("VALUE")) shouldBe Success(null) 756 | } 757 | 758 | "fails if result set is null" in { 759 | val rS = null 760 | 761 | val builder = SchemaBuilder.struct().name("test") 762 | builder.field("KEY", Schema.INT32_SCHEMA) 763 | builder.field("VALUE", Schema.OPTIONAL_STRING_SCHEMA) 764 | val schema = builder.build() 765 | 766 | val r = dataConverter.convertRecord(schema, rS) 767 | 768 | r.getClass shouldBe Failure(new NullPointerException).getClass 769 | } 770 | 771 | "fail if schema and result set don't match" in { 772 | val rS = mock[ResultSet] 773 | val metaRS = mock[ResultSetMetaData] 774 | 775 | val builder = SchemaBuilder.struct().name("test") 776 | builder.field("KEY", Schema.INT32_SCHEMA) 777 | builder.field("VALUE", Schema.BOOLEAN_SCHEMA) 778 | val schema = builder.build() 779 | 780 | when(rS.getMetaData).thenReturn(metaRS) 781 | when(metaRS.getColumnCount).thenReturn(2) 782 | when(metaRS.getColumnName(1)).thenReturn("KEY") 783 | when(metaRS.getColumnName(2)).thenReturn("VALUE") 784 | when(metaRS.getColumnLabel(1)).thenReturn("KEY") 785 | when(metaRS.getColumnLabel(2)).thenReturn("VALUE") 786 | when(metaRS.getColumnType(1)).thenReturn(Types.INTEGER) 787 | when(metaRS.getColumnType(2)).thenReturn(Types.DOUBLE) 788 | when(rS.getInt(1)).thenReturn(5) 789 | when(rS.getDouble(2)).thenReturn(3.14) 790 | 791 | val r = dataConverter.convertRecord(schema, rS) 792 | 793 | r.isFailure shouldEqual true 794 | the [DataException] thrownBy r.get 795 | } 796 | } 797 | } 798 | --------------------------------------------------------------------------------