├── LICENSE ├── README.md ├── README.zh-CN.md ├── api └── api.go ├── app └── server.go ├── build.sh ├── cmd └── qin_cdc.go ├── config ├── config.go └── plugin_config.go ├── core ├── input.go ├── meta.go ├── msg.go ├── output.go ├── position.go └── transform.go ├── docs ├── mysql-to-doris-sample.toml ├── mysql-to-kafka-sample.toml ├── mysql-to-mysql-sample.toml └── mysql-to-starrocks-sample.toml ├── go.mod ├── go.sum ├── inputs ├── init.go └── mysql │ ├── msg.go │ ├── mysql.go │ ├── mysql_meta.go │ ├── mysql_position.go │ ├── mysql_replication.go │ └── mysql_utils.go ├── metas ├── mysql_ddl_parse.go ├── routers.go └── table.go ├── metrics └── metrics.go ├── outputs ├── doris │ ├── doris.go │ ├── doris_meta.go │ └── doris_utils.go ├── init.go ├── kafka │ ├── kafka.go │ ├── kafka_meta.go │ └── kafka_utils.go ├── mysql │ ├── mysql.go │ ├── mysql_meta.go │ └── mysql_utils.go └── starrocks │ ├── starrocks.go │ ├── starrocks_meta.go │ └── starrocks_utils.go ├── registry └── registry.go ├── transforms ├── trans_delete_column.go ├── trans_rename_column.go ├── transforms.go └── utils.go └── utils ├── daemon.go ├── file_path.go ├── help.go ├── http.go ├── input_param.go └── type_cast.go /LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## qin-cdc data sync 2 | Simple, efficient, real-time, stable, scalable, highly available, AI, open source 3 | 4 | ![LICENSE](https://img.shields.io/badge/license-AGPLv3%20-blue.svg) 5 | ![](https://img.shields.io/github/languages/top/sqlpub/qin-cdc) 6 | [![Release](https://img.shields.io/github/release/sqlpub/qin-cdc.svg?style=flat-square)](https://github.com/sqlpub/qin-cdc/releases) 7 | 8 | English | [简体中文](README.zh-CN.md) 9 | 10 | ### Support sync database 11 | #### source 12 | 1. mysql 13 | 2. TODO sqlserver 14 | 3. TODO mongo 15 | 4. TODO oracle 16 | 17 | #### target 18 | 19 | 1. mysql 20 | 2. starrocks 21 | 3. doris 22 | 4. kafka json 23 | 5. TODO kafka canal 24 | 6. kafka aliyun_dts_canal 25 | 26 | ### Quick start 27 | #### 1. Install 28 | [Download](https://github.com/sqlpub/qin-cdc/releases/latest) the latest release and extract it. 29 | 30 | #### 2. Create a synchronization account 31 | ```sql 32 | CREATE USER 'qin_cdc'@'%' IDENTIFIED BY 'xxxxxx'; 33 | GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'qin_cdc'@'%'; 34 | ``` 35 | #### 3. Create a configuration file 36 | mysql-to-starrocks.toml 37 | ```toml 38 | # name is required and must be globally unique when multiple instances are running 39 | name = "mysql2starrocks" 40 | 41 | [input] 42 | type = "mysql" 43 | 44 | [input.config.source] 45 | host = "127.0.0.1" 46 | port = 3306 47 | username = "root" 48 | password = "root" 49 | 50 | [input.config.source.options] 51 | #start-gtid = "3ba13781-44eb-2157-88a5-0dc879ec2221:1-123456" 52 | #server-id = 1001 53 | 54 | [[transforms]] 55 | type = "rename-column" 56 | [transforms.config] 57 | match-schema = "mysql_test" 58 | match-table = "tb1" 59 | columns = ["col_1", "col_2"] 60 | rename-as = ["col_11", "col_22"] 61 | 62 | [[transforms]] 63 | type = "delete-column" 64 | [transforms.config] 65 | match-schema = "mysql_test" 66 | match-table = "tb1" 67 | columns = ["phone"] 68 | 69 | [output] 70 | type = "starrocks" 71 | 72 | [output.config.target] 73 | host = "127.0.0.1" 74 | port = 9030 75 | load-port = 8040 # support fe httpPort:8030 or be httpPort:8040 76 | username = "root" 77 | password = "" 78 | 79 | [input.config.target.options] 80 | batch-size = 1000 81 | batch-interval-ms = 1000 82 | parallel-workers = 4 83 | 84 | [[output.config.routers]] 85 | source-schema = "mysql_test" 86 | source-table = "tb1" 87 | target-schema = "sr_test" 88 | target-table = "ods_tb1" 89 | 90 | [[output.config.routers]] 91 | source-schema = "mysql_test" 92 | source-table = "tb2" 93 | target-schema = "sr_test" 94 | target-table = "ods_tb2" 95 | # mapper column, optional, if empty, same name mapping 96 | # [output.config.routers.columns-mapper] 97 | # source-columns = [] 98 | # target-columns = [] 99 | ``` 100 | 101 | #### 4. View Help 102 | ```shell 103 | [sr@ ~]$ ./qin-cdc-linux-xxxxxx -h 104 | ``` 105 | 106 | #### 5. Startup 107 | ```shell 108 | [sr@ ~]$ ./qin-cdc-linux-xxxxxx -config mysql-to-starrocks.toml -log-file mysql2starrocks.log -level info -daemon 109 | ``` 110 | 111 | #### 6. View logs 112 | ```shell 113 | [sr@ ~]$ tail -f mysql2starrocks.log 114 | ``` 115 | 116 | #### TODO AI functional points 117 | 1. Intelligent data synchronization and migration 118 | 2. Data security and monitoring 119 | 3. Intelligent operation and maintenance management 120 | 4. User experience optimization 121 | 5. Automated data mapping 122 | 6. Natural language processing 123 | -------------------------------------------------------------------------------- /README.zh-CN.md: -------------------------------------------------------------------------------- 1 | ## qin-cdc data sync 2 | 简单、高效、实时、稳定、可扩展、高可用、AI、开源 3 | 4 | ![LICENSE](https://img.shields.io/badge/license-AGPLv3%20-blue.svg) 5 | ![](https://img.shields.io/github/languages/top/sqlpub/qin-cdc) 6 | ![](https://img.shields.io/badge/build-release-brightgreen.svg) 7 | [![Release](https://img.shields.io/github/release/sqlpub/qin-cdc.svg?style=flat-square)](https://github.com/sqlpub/qin-cdc/releases) 8 | 9 | ### 支持同步的数据库 10 | #### 源 11 | 1. mysql 12 | 2. TODO sqlserver 13 | 3. TODO mongo 14 | 4. TODO oracle 15 | 16 | #### 目的 17 | 18 | 1. mysql 19 | 2. starrocks 20 | 3. doris 21 | 4. kafka json 22 | 5. TODO kafka canal 23 | 6. kafka aliyun_dts_canal 24 | 7. 25 | ### Quick start 26 | #### 1. 安装 27 | [Download](https://github.com/sqlpub/qin-cdc/releases/latest) the latest release and extract it. 28 | 29 | #### 2. 创建同步账号 30 | ```sql 31 | CREATE USER 'qin_cdc'@'%' IDENTIFIED BY 'xxxxxx'; 32 | GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'qin_cdc'@'%'; 33 | ``` 34 | #### 3. 创建配置文件 35 | mysql-to-starrocks.toml 36 | ```toml 37 | # name 必填，多实例运行时保证全局唯一 38 | name = "mysql2starrocks" 39 | 40 | [input] 41 | type = "mysql" 42 | 43 | [input.config.source] 44 | host = "127.0.0.1" 45 | port = 3306 46 | username = "root" 47 | password = "root" 48 | 49 | [input.config.source.options] 50 | #start-gtid = "3ba13781-44eb-2157-88a5-0dc879ec2221:1-123456" 51 | #server-id = 1001 52 | 53 | [[transforms]] 54 | type = "rename-column" 55 | [transforms.config] 56 | match-schema = "mysql_test" 57 | match-table = "tb1" 58 | columns = ["col_1", "col_2"] 59 | rename-as = ["col_11", "col_22"] 60 | 61 | [[transforms]] 62 | type = "delete-column" 63 | [transforms.config] 64 | match-schema = "mysql_test" 65 | match-table = "tb1" 66 | columns = ["phone"] 67 | 68 | [[transforms]] 69 | type = "mapper-column" 70 | [transforms.config] 71 | match-schema = "mysql_test" 72 | match-table = "tb1" 73 | [transforms.config.mapper] 74 | id = "user_id" 75 | name = "nick_name" 76 | 77 | [output] 78 | type = "starrocks" 79 | 80 | [output.config.target] 81 | host = "127.0.0.1" 82 | port = 9030 83 | load-port = 8040 # support fe httpPort:8030 or be httpPort:8040 84 | username = "root" 85 | password = "" 86 | 87 | [input.config.target.options] 88 | batch-size = 1000 89 | batch-interval-ms = 1000 90 | parallel-workers = 4 91 | 92 | [[output.config.routers]] 93 | source-schema = "sysbenchts" 94 | source-table = "sbtest1" 95 | target-schema = "sr_test" 96 | target-table = "ods_sbtest1" 97 | 98 | [[output.config.routers]] 99 | source-schema = "sysbenchts" 100 | source-table = "sbtest2" 101 | target-schema = "sr_test" 102 | target-table = "ods_sbtest2" 103 | [output.config.routers.columns-mapper] 104 | source-columns = [] 105 | target-columns = [] 106 | ``` 107 | 108 | #### 4. 查看帮助 109 | ```shell 110 | [sr@ ~]$ ./qin-cdc-linux-xxxxxx -h 111 | ``` 112 | 113 | #### 5. 启动 114 | ```shell 115 | [sr@ ~]$ ./qin-cdc-linux-xxxxxx -config mysql-to-starrocks.toml -log-file mysql2starrocks.log -level info -daemon 116 | ``` 117 | 118 | #### 6. 查看日志 119 | ```shell 120 | [sr@ ~]$ tail -f mysql2starrocks.log 121 | ``` 122 | 123 | #### TODO AI功能点 124 | 1. 智能数据同步和迁移 125 | 2. 数据安全与监控 126 | 3. 智能化运维管理 127 | 4. 用户体验优化 128 | 5. 自动化数据映射 129 | 6. 自然语言处理 -------------------------------------------------------------------------------- /api/api.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | func AddRouter() func(http.ResponseWriter, *http.Request) { 8 | return func(w http.ResponseWriter, r *http.Request) { 9 | return 10 | } 11 | } 12 | 13 | func DelRouter() func(http.ResponseWriter, *http.Request) { 14 | return func(w http.ResponseWriter, r *http.Request) { 15 | return 16 | } 17 | } 18 | 19 | func GetRouter() func(http.ResponseWriter, *http.Request) { 20 | return func(w http.ResponseWriter, r *http.Request) { 21 | return 22 | } 23 | } 24 | 25 | func PauseRouter() func(http.ResponseWriter, *http.Request) { 26 | return func(w http.ResponseWriter, r *http.Request) { 27 | return 28 | } 29 | } 30 | 31 | func ResumeRouter() func(http.ResponseWriter, *http.Request) { 32 | return func(w http.ResponseWriter, r *http.Request) { 33 | return 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /app/server.go: -------------------------------------------------------------------------------- 1 | package app 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | "github.com/sqlpub/qin-cdc/config" 6 | "github.com/sqlpub/qin-cdc/core" 7 | _ "github.com/sqlpub/qin-cdc/inputs" 8 | "github.com/sqlpub/qin-cdc/metas" 9 | _ "github.com/sqlpub/qin-cdc/outputs" 10 | "github.com/sqlpub/qin-cdc/registry" 11 | "github.com/sqlpub/qin-cdc/transforms" 12 | "sync" 13 | ) 14 | 15 | type Server struct { 16 | Input core.Input 17 | Output core.Output 18 | Metas *core.Metas 19 | Position core.Position 20 | Transforms transforms.MatcherTransforms 21 | InboundChan chan *core.Msg 22 | OutboundChan chan *core.Msg 23 | sync.Mutex 24 | } 25 | 26 | func NewServer(conf *config.Config) (server *Server, err error) { 27 | server = &Server{} 28 | 29 | // input 30 | plugin, err := registry.GetPlugin(registry.InputPlugin, conf.InputConfig.Type) 31 | if err != nil { 32 | return nil, err 33 | } 34 | input, ok := plugin.(core.Input) 35 | if !ok { 36 | return nil, errors.Errorf("not a valid input type") 37 | } 38 | server.Input = input 39 | err = plugin.Configure(conf.InputConfig.Config) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | // output 45 | plugin, err = registry.GetPlugin(registry.OutputPlugin, conf.OutputConfig.Type) 46 | if err != nil { 47 | return nil, err 48 | } 49 | output, ok := plugin.(core.Output) 50 | if !ok { 51 | return nil, errors.Errorf("not a valid output type") 52 | } 53 | server.Output = output 54 | err = plugin.Configure(conf.OutputConfig.Config) 55 | if err != nil { 56 | return nil, err 57 | } 58 | 59 | // meta 60 | err = server.initMeta(conf) 61 | if err != nil { 62 | return nil, err 63 | } 64 | 65 | // position 66 | plugin, err = registry.GetPlugin(registry.PositionPlugin, conf.InputConfig.Type) 67 | if err != nil { 68 | return nil, err 69 | } 70 | position, ok := plugin.(core.Position) 71 | if !ok { 72 | return nil, errors.Errorf("not a valid position type") 73 | } 74 | server.Position = position 75 | err = plugin.Configure(conf.InputConfig.Config) 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | // sync chan 81 | server.InboundChan = make(chan *core.Msg, 10240) 82 | server.OutboundChan = make(chan *core.Msg, 10240) 83 | 84 | // new position 85 | server.Position.LoadPosition(conf.Name) 86 | // new output 87 | server.Output.NewOutput(server.Metas) 88 | // new input 89 | server.Input.NewInput(server.Metas) 90 | 91 | return server, nil 92 | } 93 | 94 | func (s *Server) initMeta(conf *config.Config) (err error) { 95 | // Routers 96 | s.Metas = &core.Metas{ 97 | Routers: &metas.Routers{}, 98 | } 99 | err = s.Metas.Routers.InitRouters(conf.OutputConfig.Config) 100 | if err != nil { 101 | return err 102 | } 103 | 104 | // input meta 105 | plugin, err := registry.GetPlugin(registry.MetaPlugin, string(registry.InputPlugin)+conf.InputConfig.Type) 106 | if err != nil { 107 | return err 108 | } 109 | meta, ok := plugin.(core.InputMeta) 110 | if !ok { 111 | return errors.Errorf("not a valid meta type") 112 | } 113 | 114 | s.Metas.Input = meta 115 | err = plugin.Configure(conf.InputConfig.Config) 116 | if err != nil { 117 | return err 118 | } 119 | err = s.Metas.Input.LoadMeta(s.Metas.Routers.Raws) 120 | if err != nil { 121 | return err 122 | } 123 | 124 | // output meta 125 | plugin, err = registry.GetPlugin(registry.MetaPlugin, string(registry.OutputPlugin)+conf.OutputConfig.Type) 126 | if err != nil { 127 | return err 128 | } 129 | outputMeta, ok := plugin.(core.OutputMeta) 130 | if !ok { 131 | return errors.Errorf("not a valid meta type") 132 | } 133 | 134 | s.Metas.Output = outputMeta 135 | err = plugin.Configure(conf.OutputConfig.Config) 136 | if err != nil { 137 | return err 138 | } 139 | err = s.Metas.Output.LoadMeta(s.Metas.Routers.Raws) 140 | if err != nil { 141 | return err 142 | } 143 | 144 | // router column mapper 145 | err = s.Metas.InitRouterColumnsMapper() 146 | if err != nil { 147 | return err 148 | } 149 | 150 | // trans 151 | s.Transforms = transforms.NewMatcherTransforms(conf.TransformsConfig, s.Metas.Routers) 152 | 153 | // router column mapper MapMapper 154 | s.Metas.InitRouterColumnsMapperMapMapper() 155 | return nil 156 | } 157 | 158 | func (s *Server) Start() { 159 | s.Lock() 160 | defer s.Unlock() 161 | 162 | s.Input.Start(s.Position, s.InboundChan) 163 | s.Position.Start() 164 | s.Transforms.Start(s.InboundChan, s.OutboundChan) 165 | s.Output.Start(s.OutboundChan, s.Position) 166 | } 167 | 168 | func (s *Server) Close() { 169 | s.Lock() 170 | defer s.Unlock() 171 | 172 | s.Input.Close() 173 | s.Output.Close() 174 | s.Position.Close() 175 | s.Metas.Input.Close() 176 | s.Metas.Output.Close() 177 | } 178 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | version="v0.3.0" 4 | currentDir=$(cd $(dirname "$0") || exit; pwd) 5 | 6 | path="github.com/go-demo/version" 7 | buildTime=$(date +"%Y-%m-%d %H:%M:%S") 8 | buildTimeFormat=$(date +"%Y%m%d%H%M%S") 9 | newDir="../bin/qin-cdc-$version" 10 | # flagsMac="-X $path.Version=$version -X '$path.GoVersion=$(go version)' -X '$path.BuildTime=$buildTime' -X $path.GitCommit=$(git rev-parse HEAD)" 11 | flagsLinux="-X $path.Version=$version -X '$path.GoVersion=$(go version)' -X '$path.BuildTime=$buildTime' -X $path.GitCommit=$(git rev-parse HEAD)" 12 | 13 | mkdir -p "$newDir" 14 | echo start buid qin-cdc 15 | cd "$currentDir"/cmd || exit 16 | # go build -ldflags "$flagsMac" -o "$newDir"/go-"$dbType"-starrocks-mac-"$buildTimeFormat" 17 | CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC=x86_64-linux-musl-gcc CXX=x86_64-linux-musl-g++ \ 18 | go build -tags musl -ldflags "-extldflags -static $flagsLinux" -o "$newDir"/qin-cdc-$version-"$buildTimeFormat" 19 | echo end buid qin-cdc -------------------------------------------------------------------------------- /cmd/qin_cdc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/siddontang/go-log/log" 5 | "github.com/sqlpub/qin-cdc/app" 6 | "github.com/sqlpub/qin-cdc/config" 7 | "github.com/sqlpub/qin-cdc/utils" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | ) 12 | 13 | func main() { 14 | // help handle 15 | inputParam := utils.InitHelp() 16 | // input param handle 17 | utils.InputParamHandle(inputParam) 18 | // init log level 19 | log.SetLevelByName(*inputParam.LogLevel) 20 | // daemon mode handle 21 | utils.Daemon(inputParam) 22 | 23 | // 进程信号处理 24 | sc := make(chan os.Signal, 1) 25 | signal.Notify(sc, 26 | os.Kill, 27 | os.Interrupt, 28 | syscall.SIGHUP, 29 | syscall.SIGINT, 30 | syscall.SIGTERM, 31 | syscall.SIGQUIT) 32 | 33 | utils.StartHttp(inputParam) 34 | 35 | // config file handle 36 | conf := config.NewConfig(inputParam.ConfigFile) 37 | s, err := app.NewServer(conf) 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | s.Start() 42 | 43 | utils.InitHttpApi() 44 | 45 | select { 46 | case n := <-sc: 47 | log.Infof("receive signal %v, closing", n) 48 | s.Close() 49 | log.Infof("qin-cdc is stopped.") 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/BurntSushi/toml" 5 | "github.com/juju/errors" 6 | "github.com/siddontang/go-log/log" 7 | "path/filepath" 8 | ) 9 | 10 | type Config struct { 11 | Name string 12 | InputConfig InputConfig `toml:"input"` 13 | OutputConfig OutputConfig `toml:"output"` 14 | TransformsConfig []TransformConfig `toml:"transforms"` 15 | FileName *string 16 | } 17 | 18 | type InputConfig struct { 19 | Type string `toml:"type"` 20 | Config map[string]interface{} `toml:"config"` 21 | } 22 | 23 | type OutputConfig struct { 24 | Type string `toml:"type"` 25 | Config map[string]interface{} `toml:"config"` 26 | } 27 | 28 | type TransformConfig struct { 29 | Type string `toml:"type"` 30 | Config map[string]interface{} `toml:"config"` 31 | } 32 | 33 | func NewConfig(fileName *string) (c *Config) { 34 | c = &Config{} 35 | fileNamePath, err := filepath.Abs(*fileName) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | c.FileName = &fileNamePath 40 | err = c.ReadConfig() 41 | if err != nil { 42 | log.Fatal(err) 43 | } 44 | return c 45 | } 46 | 47 | func (c *Config) ReadConfig() error { 48 | var err error 49 | if _, err = toml.DecodeFile(*c.FileName, c); err != nil { 50 | return errors.Trace(err) 51 | } 52 | return err 53 | } 54 | -------------------------------------------------------------------------------- /config/plugin_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type MysqlConfig struct { 4 | Host string 5 | Port int 6 | UserName string 7 | Password string 8 | Options struct { 9 | StartGtid string `toml:"start-gtid"` 10 | ServerId int `toml:"server-id"` 11 | BatchSize int `toml:"batch-size"` 12 | BatchIntervalMs int `toml:"batch-interval-ms"` 13 | } 14 | } 15 | 16 | type StarrocksConfig struct { 17 | Host string 18 | Port int 19 | LoadPort int `mapstructure:"load-port"` 20 | UserName string 21 | Password string 22 | Options struct { 23 | BatchSize int `toml:"batch-size" mapstructure:"batch-size"` 24 | BatchIntervalMs int `toml:"batch-interval-ms" mapstructure:"batch-interval-ms"` 25 | } 26 | } 27 | 28 | type DorisConfig struct { 29 | Host string 30 | Port int 31 | LoadPort int `mapstructure:"load-port"` 32 | UserName string 33 | Password string 34 | Options struct { 35 | BatchSize int `toml:"batch-size" mapstructure:"batch-size"` 36 | BatchIntervalMs int `toml:"batch-interval-ms" mapstructure:"batch-interval-ms"` 37 | } 38 | } 39 | 40 | type KafkaConfig struct { 41 | Brokers []string `toml:"brokers"` 42 | PartitionNum int `toml:"partition-num" mapstructure:"partition-num"` 43 | Options struct { 44 | BatchSize int `toml:"batch-size" mapstructure:"batch-size"` 45 | BatchIntervalMs int `toml:"batch-interval-ms" mapstructure:"batch-interval-ms"` 46 | OutputFormat string `toml:"output-format" mapstructure:"output-format"` 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /core/input.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type Input interface { 4 | NewInput(metas *Metas) 5 | Start(pos Position, in chan *Msg) 6 | Close() 7 | } 8 | -------------------------------------------------------------------------------- /core/meta.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | "github.com/sqlpub/qin-cdc/metas" 6 | ) 7 | 8 | type InputMeta interface { 9 | LoadMeta(routers []*metas.Router) error 10 | GetMeta(*metas.Router) (*metas.Table, error) 11 | // GetAll() map[string]*metas.Table 12 | GetVersion(schema string, tableName string, version uint) (*metas.Table, error) 13 | // Add(*metas.Table) error 14 | // Update(newTable *metas.Table) error 15 | // Delete(string, string) error 16 | Save() error 17 | Close() 18 | } 19 | 20 | type OutputMeta interface { 21 | LoadMeta(routers []*metas.Router) error 22 | GetMeta(*metas.Router) (interface{}, error) 23 | // GetAll() map[string]*metas.Table 24 | // GetVersion(schema string, tableName string, version uint) (*metas.Table, error) 25 | // Add(*metas.Table) error 26 | // Update(newTable *metas.Table) error 27 | // Delete(string, string) error 28 | Save() error 29 | Close() 30 | } 31 | 32 | type Metas struct { 33 | Input InputMeta 34 | Output OutputMeta 35 | Routers *metas.Routers 36 | } 37 | 38 | func (m *Metas) InitRouterColumnsMapper() error { 39 | // router column mapper 40 | for _, router := range m.Routers.Raws { 41 | inputTable, err := m.Input.GetMeta(router) 42 | if err != nil { 43 | return err 44 | } 45 | if inputTable == nil { 46 | return errors.Errorf("get input meta failed, err: %s.%s not found", router.SourceSchema, router.SourceTable) 47 | } 48 | for _, column := range inputTable.Columns { 49 | router.ColumnsMapper.SourceColumns = append(router.ColumnsMapper.SourceColumns, column.Name) 50 | if column.IsPrimaryKey { 51 | router.ColumnsMapper.PrimaryKeys = append(router.ColumnsMapper.PrimaryKeys, column.Name) 52 | } 53 | } 54 | metaObj, err := m.Output.GetMeta(router) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | outputTable, ok := metaObj.(*metas.Table) 60 | if ok { 61 | if outputTable == nil { 62 | return errors.Errorf("get output meta failed, err: %s.%s not found", router.TargetSchema, router.TargetTable) 63 | } 64 | for _, column := range outputTable.Columns { 65 | router.ColumnsMapper.TargetColumns = append(router.ColumnsMapper.TargetColumns, column.Name) 66 | } 67 | } else { 68 | // target == source 69 | for _, column := range inputTable.Columns { 70 | router.ColumnsMapper.TargetColumns = append(router.ColumnsMapper.TargetColumns, column.Name) 71 | } 72 | } 73 | 74 | } 75 | return nil 76 | } 77 | 78 | func (m *Metas) InitRouterColumnsMapperMapMapper() { 79 | // router column mapper MapMapper 80 | for _, router := range m.Routers.Raws { 81 | mapMapper := make(map[string]string) 82 | mapMapperOrder := make([]string, 0) 83 | // user config output.config.routers.columns-mapper.map-mapper 84 | if len(router.ColumnsMapper.MapMapper) > 0 { 85 | for i, column := range router.ColumnsMapper.SourceColumns { 86 | mapMapper[column] = router.ColumnsMapper.TargetColumns[i] 87 | mapMapperOrder = append(mapMapperOrder, column) 88 | } 89 | } else { 90 | for _, column := range router.ColumnsMapper.SourceColumns { 91 | // same name mapping 92 | for _, targetColumn := range router.ColumnsMapper.TargetColumns { 93 | if column == targetColumn { 94 | mapMapper[column] = targetColumn 95 | mapMapperOrder = append(mapMapperOrder, column) 96 | break 97 | } 98 | } 99 | } 100 | } 101 | router.ColumnsMapper.MapMapper = mapMapper 102 | router.ColumnsMapper.MapMapperOrder = mapMapperOrder 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /core/msg.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | "github.com/goccy/go-json" 6 | "github.com/sqlpub/qin-cdc/metas" 7 | "time" 8 | ) 9 | 10 | type MsgType string 11 | type ActionType string 12 | type DDLActionType string 13 | 14 | const ( 15 | MsgDML MsgType = "dml" 16 | MsgDDL MsgType = "ddl" 17 | MsgCtl MsgType = "ctl" 18 | 19 | InsertAction ActionType = "insert" 20 | UpdateAction ActionType = "update" 21 | DeleteAction ActionType = "delete" 22 | ReplaceAction ActionType = "replace" 23 | 24 | CreateAction DDLActionType = "create" 25 | AlterAction DDLActionType = "alter" 26 | RenameAction DDLActionType = "rename" 27 | DropAction DDLActionType = "drop" 28 | TruncateAction DDLActionType = "Truncate" 29 | ) 30 | 31 | type Msg struct { 32 | Database string 33 | Table string 34 | Type MsgType 35 | DmlMsg *DMLMsg 36 | Timestamp time.Time 37 | InputContext struct { 38 | Pos string 39 | } 40 | } 41 | 42 | type DMLMsg struct { 43 | Action ActionType 44 | Data map[string]interface{} 45 | Old map[string]interface{} 46 | TableVersion uint 47 | } 48 | 49 | type DDLMsg struct { 50 | Action DDLActionType 51 | NewTable metas.Table 52 | DdlStatement metas.DdlStatement 53 | } 54 | 55 | func (m *Msg) ToString() string { 56 | switch m.Type { 57 | case MsgDML: 58 | marshal, _ := json.Marshal(m.DmlMsg) 59 | return fmt.Sprintf("msg event: %s %s.%s %v", m.DmlMsg.Action, m.Database, m.Table, string(marshal)) 60 | case MsgDDL: 61 | case MsgCtl: 62 | marshal, _ := json.Marshal(m.InputContext) 63 | return fmt.Sprintf("msg event: %s %v", m.Type, string(marshal)) 64 | default: 65 | return fmt.Sprintf("msg event: %s %v", m.DmlMsg.Action, m) 66 | } 67 | return "" 68 | } 69 | -------------------------------------------------------------------------------- /core/output.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type Output interface { 4 | NewOutput(metas *Metas) 5 | Start(out chan *Msg, pos Position) 6 | Close() 7 | } 8 | -------------------------------------------------------------------------------- /core/position.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type Position interface { 4 | LoadPosition(name string) string 5 | Start() 6 | Update(v string) error 7 | Save() error 8 | Get() string 9 | Close() 10 | } 11 | -------------------------------------------------------------------------------- /core/transform.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type Transform interface { 4 | NewTransform(config map[string]interface{}) error 5 | Transform(msg *Msg) bool 6 | } 7 | -------------------------------------------------------------------------------- /docs/mysql-to-doris-sample.toml: -------------------------------------------------------------------------------- 1 | # name 必填，多实例运行时保证全局唯一 2 | name = "mysql2doris" 3 | 4 | [input] 5 | type = "mysql" 6 | 7 | [input.config.source] 8 | host = "127.0.0.1" 9 | port = 3306 10 | username = "root" 11 | password = "root" 12 | 13 | [input.config.source.options] 14 | #start-gtid = "3ba13781-44eb-2157-88a5-0dc879ec2221:1-123456" 15 | #server-id = 1001 16 | 17 | [[transforms]] 18 | type = "rename-column" 19 | [transforms.config] 20 | match-schema = "sysbenchts" 21 | match-table = "sbtest1" 22 | columns = ["k", "c"] 23 | rename-as = ["k_1", "c_1"] 24 | 25 | [[transforms]] 26 | type = "delete-column" 27 | [transforms.config] 28 | match-schema = "sysbenchts" 29 | match-table = "sbtest1" 30 | columns = ["c_1"] 31 | 32 | [output] 33 | type = "doris" 34 | 35 | [output.config.target] 36 | host = "127.0.0.1" 37 | port = 9030 38 | load-port = 8030 # support fe httpPort:8030 or be httpPort:8040 39 | username = "root" 40 | password = "root" 41 | 42 | [input.config.target.options] 43 | batch-size = 1000 44 | batch-interval-ms = 1000 45 | parallel-workers = 4 46 | 47 | [[output.config.routers]] 48 | source-schema = "sysbenchts" 49 | source-table = "sbtest1" 50 | target-schema = "doris_test" 51 | target-table = "ods_sbtest1" 52 | 53 | [[output.config.routers]] 54 | source-schema = "sysbenchts" 55 | source-table = "sbtest2" 56 | target-schema = "doris_test" 57 | target-table = "ods_sbtest2" 58 | [output.config.routers.columns-mapper] 59 | source-columns = [] 60 | target-columns = [] -------------------------------------------------------------------------------- /docs/mysql-to-kafka-sample.toml: -------------------------------------------------------------------------------- 1 | # name 必填，多实例运行时保证全局唯一 2 | name = "mysql2kafka" 3 | 4 | [input] 5 | type = "mysql" 6 | 7 | [input.config.source] 8 | host = "127.0.0.1" 9 | port = 3306 10 | username = "root" 11 | password = "root" 12 | 13 | [input.config.source.options] 14 | #start-gtid = "3ba13781-44eb-2157-88a5-0dc879ec2221:1-123456" 15 | #server-id = 1001 16 | 17 | [[transforms]] 18 | type = "rename-column" 19 | [transforms.config] 20 | match-schema = "sysbenchts" 21 | match-table = "sbtest1" 22 | columns = ["k", "c"] 23 | rename-as = ["k_1", "c_1"] 24 | 25 | [[transforms]] 26 | type = "delete-column" 27 | [transforms.config] 28 | match-schema = "sysbenchts" 29 | match-table = "sbtest1" 30 | columns = ["c_1"] 31 | 32 | [output] 33 | type = "kafka" 34 | 35 | [output.config.target] 36 | brokers = ["127.0.0.1:9092"] 37 | partition-num = 1 38 | 39 | [output.config.target.options] 40 | batch-size = 1000 41 | batch-interval-ms = 1000 42 | parallel-workers = 4 43 | output-format = "json" # or aliyun_dts_canal 44 | 45 | [[output.config.routers]] 46 | source-schema = "sysbenchts" 47 | source-table = "sbtest1" 48 | dml-topic = "mysql-binlog" 49 | [output.config.routers.columns-mapper] 50 | source-columns = [] 51 | target-columns = [] 52 | 53 | [[output.config.routers]] 54 | source-schema = "sysbenchts" 55 | source-table = "sbtest2" 56 | dml-topic = "mysql-binlog" 57 | [output.config.routers.columns-mapper] 58 | source-columns = [] 59 | target-columns = [] -------------------------------------------------------------------------------- /docs/mysql-to-mysql-sample.toml: -------------------------------------------------------------------------------- 1 | # name 必填，多实例运行时保证全局唯一 2 | name = "mysql2mysql" 3 | 4 | [input] 5 | type = "mysql" 6 | 7 | [input.config.source] 8 | host = "127.0.0.1" 9 | port = 3306 10 | username = "root" 11 | password = "root" 12 | 13 | [input.config.source.options] 14 | #start-gtid = "3ba13781-44eb-2157-88a5-0dc879ec2221:1-123456" 15 | #server-id = 1001 16 | 17 | [[transforms]] 18 | type = "rename-column" 19 | [transforms.config] 20 | match-schema = "sysbenchts" 21 | match-table = "sbtest1" 22 | columns = ["c"] 23 | rename-as = ["c_1"] 24 | 25 | [[transforms]] 26 | type = "delete-column" 27 | [transforms.config] 28 | match-schema = "sysbenchts" 29 | match-table = "sbtest1" 30 | columns = ["c_1"] 31 | 32 | [output] 33 | type = "mysql" 34 | 35 | [output.config.target] 36 | host = "127.0.0.1" 37 | port = 3307 38 | username = "root" 39 | password = "root" 40 | 41 | [input.config.target.options] 42 | batch-size = 1000 43 | batch-interval-ms = 500 44 | parallel-workers = 4 45 | 46 | [[output.config.routers]] 47 | source-schema = "sysbenchts" 48 | source-table = "sbtest1" 49 | target-schema = "sysbenchts" 50 | target-table = "sbtest1" 51 | 52 | [[output.config.routers]] 53 | source-schema = "sysbenchts" 54 | source-table = "sbtest2" 55 | target-schema = "sysbenchts" 56 | target-table = "sbtest2" 57 | [output.config.routers.columns-mapper] 58 | source-columns = [] 59 | target-columns = [] -------------------------------------------------------------------------------- /docs/mysql-to-starrocks-sample.toml: -------------------------------------------------------------------------------- 1 | # name 必填，多实例运行时保证全局唯一 2 | name = "mysql2starrocks" 3 | 4 | [input] 5 | type = "mysql" 6 | 7 | [input.config.source] 8 | host = "127.0.0.1" 9 | port = 3306 10 | username = "root" 11 | password = "root" 12 | 13 | [input.config.source.options] 14 | #start-gtid = "3ba13781-44eb-2157-88a5-0dc879ec2221:1-123456" 15 | #server-id = 1001 16 | 17 | [[transforms]] 18 | type = "rename-column" 19 | [transforms.config] 20 | match-schema = "sysbenchts" 21 | match-table = "sbtest1" 22 | columns = ["k", "c"] 23 | rename-as = ["k_1", "c_1"] 24 | 25 | [[transforms]] 26 | type = "delete-column" 27 | [transforms.config] 28 | match-schema = "sysbenchts" 29 | match-table = "sbtest1" 30 | columns = ["c_1"] 31 | 32 | [output] 33 | type = "starrocks" 34 | 35 | [output.config.target] 36 | host = "127.0.0.1" 37 | port = 9030 38 | load-port = 8040 # support fe httpPort:8030 or be httpPort:8040 39 | username = "root" 40 | password = "" 41 | 42 | [input.config.target.options] 43 | batch-size = 1000 44 | batch-interval-ms = 1000 45 | parallel-workers = 4 46 | 47 | [[output.config.routers]] 48 | source-schema = "sysbenchts" 49 | source-table = "sbtest1" 50 | target-schema = "sr_test" 51 | target-table = "ods_sbtest1" 52 | 53 | [[output.config.routers]] 54 | source-schema = "sysbenchts" 55 | source-table = "sbtest2" 56 | target-schema = "sr_test" 57 | target-table = "ods_sbtest2" 58 | [output.config.routers.columns-mapper] 59 | source-columns = [] 60 | target-columns = [] -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/sqlpub/qin-cdc 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/BurntSushi/toml v1.4.0 7 | github.com/confluentinc/confluent-kafka-go/v2 v2.4.0 8 | github.com/go-demo/version v0.0.0-20200109120206-2cde9473fd92 9 | github.com/go-mysql-org/go-mysql v1.8.0 10 | github.com/go-sql-driver/mysql v1.8.1 11 | github.com/goccy/go-json v0.10.3 12 | github.com/google/uuid v1.6.0 13 | github.com/juju/errors v1.0.0 14 | github.com/mitchellh/hashstructure/v2 v2.0.2 15 | github.com/mitchellh/mapstructure v1.5.0 16 | github.com/pingcap/tidb/pkg/parser v0.0.0-20240516062813-cc127c14b8cc 17 | github.com/prometheus/client_golang v1.19.1 18 | github.com/sevlyar/go-daemon v0.1.6 19 | github.com/siddontang/go-log v0.0.0-20190221022429-1e957dd83bed 20 | go.etcd.io/bbolt v1.3.10 21 | ) 22 | 23 | require ( 24 | filippo.io/edwards25519 v1.1.0 // indirect 25 | github.com/Masterminds/semver v1.5.0 // indirect 26 | github.com/beorn7/perks v1.0.1 // indirect 27 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 28 | github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect 29 | github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect 30 | github.com/klauspost/compress v1.17.4 // indirect 31 | github.com/pingcap/errors v0.11.5-0.20240311024730-e056997136bb // indirect 32 | github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect 33 | github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 // indirect 34 | github.com/prometheus/client_model v0.5.0 // indirect 35 | github.com/prometheus/common v0.48.0 // indirect 36 | github.com/prometheus/procfs v0.12.0 // indirect 37 | github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect 38 | github.com/shopspring/decimal v1.2.0 // indirect 39 | github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 // indirect 40 | go.uber.org/atomic v1.11.0 // indirect 41 | go.uber.org/multierr v1.11.0 // indirect 42 | go.uber.org/zap v1.26.0 // indirect 43 | golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect 44 | golang.org/x/sys v0.18.0 // indirect 45 | golang.org/x/text v0.14.0 // indirect 46 | google.golang.org/protobuf v1.33.0 // indirect 47 | gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect 48 | ) 49 | -------------------------------------------------------------------------------- /inputs/init.go: -------------------------------------------------------------------------------- 1 | package inputs 2 | 3 | import ( 4 | "github.com/sqlpub/qin-cdc/inputs/mysql" 5 | "github.com/sqlpub/qin-cdc/registry" 6 | ) 7 | 8 | func init() { 9 | // input mysql plugins 10 | registry.RegisterPlugin(registry.InputPlugin, mysql.PluginName, &mysql.InputPlugin{}) 11 | registry.RegisterPlugin(registry.MetaPlugin, string(registry.InputPlugin+mysql.PluginName), &mysql.MetaPlugin{}) 12 | registry.RegisterPlugin(registry.PositionPlugin, mysql.PluginName, &mysql.PositionPlugin{}) 13 | } 14 | -------------------------------------------------------------------------------- /inputs/mysql/msg.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "github.com/go-mysql-org/go-mysql/replication" 5 | "github.com/sqlpub/qin-cdc/core" 6 | "github.com/sqlpub/qin-cdc/metas" 7 | "time" 8 | ) 9 | 10 | func (i *InputPlugin) NewInsertMsgs(schemaName string, tableName string, tableMeta *metas.Table, evs *replication.RowsEvent, header *replication.EventHeader) (msgs []*core.Msg, err error) { 11 | // new insert msgs 12 | msgs = make([]*core.Msg, len(evs.Rows)) 13 | for index, row := range evs.Rows { 14 | data := make(map[string]interface{}) 15 | for columnIndex, value := range row { 16 | data[tableMeta.Columns[columnIndex].Name] = deserialize(value, tableMeta.Columns[columnIndex]) 17 | } 18 | msg := &core.Msg{ 19 | Database: schemaName, 20 | Table: tableName, 21 | Type: core.MsgDML, 22 | DmlMsg: &core.DMLMsg{Action: core.InsertAction, Data: data, TableVersion: tableMeta.Version}, 23 | Timestamp: time.Unix(int64(header.Timestamp), 0), 24 | } 25 | msgs[index] = msg 26 | } 27 | return msgs, nil 28 | } 29 | 30 | func (i *InputPlugin) NewUpdateMsgs(schemaName string, tableName string, tableMeta *metas.Table, evs *replication.RowsEvent, header *replication.EventHeader) (msgs []*core.Msg, err error) { 31 | // new update msgs 32 | for index := 0; index < len(evs.Rows); index += 2 { 33 | oldDataRow := evs.Rows[index] 34 | newDataRow := evs.Rows[index+1] 35 | 36 | data := make(map[string]interface{}) 37 | old := make(map[string]interface{}) 38 | 39 | for columnIndex := 0; columnIndex < len(newDataRow); columnIndex++ { 40 | data[tableMeta.Columns[columnIndex].Name] = deserialize(newDataRow[columnIndex], tableMeta.Columns[columnIndex]) 41 | old[tableMeta.Columns[columnIndex].Name] = deserialize(oldDataRow[columnIndex], tableMeta.Columns[columnIndex]) 42 | } 43 | msg := &core.Msg{ 44 | Database: schemaName, 45 | Table: tableName, 46 | Type: core.MsgDML, 47 | DmlMsg: &core.DMLMsg{Action: core.UpdateAction, Data: data, Old: old, TableVersion: tableMeta.Version}, 48 | Timestamp: time.Unix(int64(header.Timestamp), 0), 49 | } 50 | msgs = append(msgs, msg) 51 | 52 | } 53 | return msgs, nil 54 | } 55 | 56 | func (i *InputPlugin) NewDeleteMsgs(schemaName string, tableName string, tableMeta *metas.Table, evs *replication.RowsEvent, header *replication.EventHeader) (msgs []*core.Msg, err error) { 57 | // new delete msgs 58 | msgs = make([]*core.Msg, len(evs.Rows)) 59 | for index, row := range evs.Rows { 60 | data := make(map[string]interface{}) 61 | for columnIndex, value := range row { 62 | data[tableMeta.Columns[columnIndex].Name] = deserialize(value, tableMeta.Columns[columnIndex]) 63 | } 64 | msg := &core.Msg{ 65 | Database: schemaName, 66 | Table: tableName, 67 | Type: core.MsgDML, 68 | DmlMsg: &core.DMLMsg{Action: core.DeleteAction, Data: data, TableVersion: tableMeta.Version}, 69 | Timestamp: time.Unix(int64(header.Timestamp), 0), 70 | } 71 | msgs[index] = msg 72 | } 73 | return msgs, nil 74 | } 75 | 76 | func (i *InputPlugin) NewXIDMsg(ev *replication.XIDEvent, header *replication.EventHeader) (msg *core.Msg, err error) { 77 | // new xid msg 78 | msg = &core.Msg{ 79 | Type: core.MsgCtl, 80 | Timestamp: time.Unix(int64(header.Timestamp), 0), 81 | } 82 | msg.InputContext.Pos = ev.GSet.String() 83 | return msg, nil 84 | } 85 | 86 | func (i *InputPlugin) SendMsgs(msgs []*core.Msg) { 87 | for _, msg := range msgs { 88 | i.SendMsg(msg) 89 | } 90 | } 91 | 92 | func (i *InputPlugin) SendMsg(msg *core.Msg) { 93 | i.in <- msg 94 | } 95 | -------------------------------------------------------------------------------- /inputs/mysql/mysql.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "github.com/mitchellh/mapstructure" 5 | "github.com/siddontang/go-log/log" 6 | "github.com/sqlpub/qin-cdc/config" 7 | "github.com/sqlpub/qin-cdc/core" 8 | ) 9 | 10 | type InputPlugin struct { 11 | *config.MysqlConfig 12 | in chan *core.Msg 13 | metas *core.Metas 14 | metaPlugin *MetaPlugin 15 | binlogTailer *BinlogTailer 16 | } 17 | 18 | func (i *InputPlugin) Configure(conf map[string]interface{}) error { 19 | i.MysqlConfig = &config.MysqlConfig{} 20 | var source = conf["source"] 21 | if err := mapstructure.Decode(source, i.MysqlConfig); err != nil { 22 | return err 23 | } 24 | return nil 25 | } 26 | 27 | func (i *InputPlugin) NewInput(metas *core.Metas) { 28 | i.metas = metas 29 | i.metaPlugin = metas.Input.(*MetaPlugin) 30 | } 31 | 32 | func (i *InputPlugin) Start(pos core.Position, in chan *core.Msg) { 33 | i.in = in 34 | var positionPlugin = &PositionPlugin{} 35 | if err := mapstructure.Decode(pos, positionPlugin); err != nil { 36 | log.Fatalf("mysql position parsing failed. err: %s", err.Error()) 37 | } 38 | 39 | i.binlogTailer = &BinlogTailer{} 40 | i.binlogTailer.New(i) 41 | go i.binlogTailer.Start(positionPlugin.pos) 42 | } 43 | 44 | func (i *InputPlugin) Close() { 45 | i.binlogTailer.Close() 46 | close(i.in) 47 | } 48 | -------------------------------------------------------------------------------- /inputs/mysql/mysql_meta.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | _ "github.com/go-sql-driver/mysql" 7 | "github.com/mitchellh/mapstructure" 8 | "github.com/sqlpub/qin-cdc/config" 9 | "github.com/sqlpub/qin-cdc/metas" 10 | "strings" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | type MetaPlugin struct { 16 | *config.MysqlConfig 17 | tables map[string]*metas.Table 18 | tablesVersion map[string]*metas.Table 19 | db *sql.DB 20 | mu sync.Mutex 21 | } 22 | 23 | func (m *MetaPlugin) Configure(conf map[string]interface{}) error { 24 | m.MysqlConfig = &config.MysqlConfig{} 25 | var source = conf["source"] 26 | if err := mapstructure.Decode(source, m.MysqlConfig); err != nil { 27 | return err 28 | } 29 | return nil 30 | } 31 | 32 | func (m *MetaPlugin) LoadMeta(routers []*metas.Router) (err error) { 33 | m.tables = make(map[string]*metas.Table) 34 | m.tablesVersion = make(map[string]*metas.Table) 35 | dsn := fmt.Sprintf( 36 | "%s:%s@tcp(%s:%d)/information_schema?charset=utf8mb4&timeout=3s", 37 | m.MysqlConfig.UserName, m.MysqlConfig.Password, 38 | m.MysqlConfig.Host, m.MysqlConfig.Port) 39 | m.db, err = sql.Open("mysql", dsn) 40 | if err != nil { 41 | return err 42 | } 43 | m.db.SetConnMaxLifetime(time.Minute * 3) 44 | m.db.SetMaxOpenConns(2) 45 | m.db.SetMaxIdleConns(2) 46 | for _, router := range routers { 47 | row := m.db.QueryRow(fmt.Sprintf("show create table `%s`.`%s`", router.SourceSchema, router.SourceTable)) 48 | if row.Err() != nil { 49 | return err 50 | } 51 | var tableName string 52 | var createTableDdlStr string 53 | err = row.Scan(&tableName, &createTableDdlStr) 54 | if err != nil { 55 | return err 56 | } 57 | createTableDdlStr = strings.Replace(createTableDdlStr, "CREATE TABLE ", fmt.Sprintf("CREATE TABLE `%s`.", router.SourceSchema), 1) 58 | table := &metas.Table{} 59 | table, err = metas.NewTable(createTableDdlStr) 60 | if err != nil { 61 | return err 62 | } 63 | err = m.Add(table) 64 | if err != nil { 65 | return err 66 | } 67 | } 68 | return nil 69 | } 70 | 71 | func (m *MetaPlugin) GetMeta(router *metas.Router) (table *metas.Table, err error) { 72 | return m.Get(router.SourceSchema, router.SourceTable) 73 | } 74 | 75 | func (m *MetaPlugin) Get(schema string, tableName string) (table *metas.Table, err error) { 76 | key := metas.GenerateMapRouterKey(schema, tableName) 77 | m.mu.Lock() 78 | defer m.mu.Unlock() 79 | return m.tables[key], err 80 | } 81 | 82 | func (m *MetaPlugin) GetAll() map[string]*metas.Table { 83 | m.mu.Lock() 84 | defer m.mu.Unlock() 85 | return m.tables 86 | } 87 | 88 | func (m *MetaPlugin) GetVersion(schema string, tableName string, version uint) (table *metas.Table, err error) { 89 | key := metas.GenerateMapRouterVersionKey(schema, tableName, version) 90 | m.mu.Lock() 91 | defer m.mu.Unlock() 92 | return m.tablesVersion[key], err 93 | } 94 | 95 | func (m *MetaPlugin) GetVersions(schema string, tableName string) []*metas.Table { 96 | m.mu.Lock() 97 | defer m.mu.Unlock() 98 | tables := make([]*metas.Table, 0) 99 | for k, table := range m.tablesVersion { 100 | s, t, _ := metas.SplitMapRouterVersionKey(k) 101 | if schema == s && tableName == t { 102 | tables = append(tables, table) 103 | } 104 | } 105 | return tables 106 | } 107 | 108 | func (m *MetaPlugin) Add(newTable *metas.Table) error { 109 | m.mu.Lock() 110 | defer m.mu.Unlock() 111 | m.tables[metas.GenerateMapRouterKey(newTable.Schema, newTable.Name)] = newTable 112 | m.tablesVersion[metas.GenerateMapRouterVersionKey(newTable.Schema, newTable.Name, newTable.Version)] = newTable 113 | return nil 114 | } 115 | 116 | func (m *MetaPlugin) Update(newTable *metas.Table) error { 117 | m.mu.Lock() 118 | defer m.mu.Unlock() 119 | newTable.Version += 1 120 | m.tables[metas.GenerateMapRouterKey(newTable.Schema, newTable.Name)] = newTable 121 | m.tablesVersion[metas.GenerateMapRouterVersionKey(newTable.Schema, newTable.Name, newTable.Version)] = newTable 122 | return nil 123 | } 124 | 125 | func (m *MetaPlugin) Delete(schema string, name string) error { 126 | m.mu.Lock() 127 | defer m.mu.Unlock() 128 | delete(m.tables, metas.GenerateMapRouterKey(schema, name)) 129 | for _, table := range m.GetVersions(schema, name) { 130 | delete(m.tablesVersion, metas.GenerateMapRouterVersionKey(schema, name, table.Version)) 131 | } 132 | return nil 133 | } 134 | 135 | func (m *MetaPlugin) Save() error { 136 | return nil 137 | } 138 | 139 | func (m *MetaPlugin) Close() { 140 | if m.db != nil { 141 | _ = m.db.Close() 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /inputs/mysql/mysql_position.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "fmt" 5 | "github.com/juju/errors" 6 | "github.com/mitchellh/mapstructure" 7 | "github.com/siddontang/go-log/log" 8 | "github.com/sqlpub/qin-cdc/config" 9 | bolt "go.etcd.io/bbolt" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | type PositionPlugin struct { 15 | *config.MysqlConfig 16 | name string 17 | metaDb *bolt.DB 18 | bucketName string 19 | pos string 20 | stop chan struct{} 21 | done chan struct{} 22 | mu sync.Mutex 23 | } 24 | 25 | func (p *PositionPlugin) Configure(conf map[string]interface{}) error { 26 | p.MysqlConfig = &config.MysqlConfig{} 27 | var source = conf["source"] 28 | if err := mapstructure.Decode(source, p.MysqlConfig); err != nil { 29 | return err 30 | } 31 | p.bucketName = "position" 32 | p.stop = make(chan struct{}) 33 | p.done = make(chan struct{}) 34 | return nil 35 | } 36 | 37 | func (p *PositionPlugin) LoadPosition(name string) string { 38 | p.name = name 39 | var err error 40 | p.metaDb, err = bolt.Open("meta.db", 0600, &bolt.Options{Timeout: 1 * time.Second}) 41 | if err != nil { 42 | log.Fatal(err) 43 | } 44 | p.pos = p.getPosFromMetaDb() // meta.db 45 | if p.pos != "" { 46 | return p.pos 47 | } 48 | p.pos = p.getPosFromConfig() // config file 49 | if p.pos != "" { 50 | return p.pos 51 | } 52 | p.pos = p.getPosFromSource() // from db get now position 53 | return p.pos 54 | } 55 | 56 | func (p *PositionPlugin) Start() { 57 | go p.timerSave() 58 | } 59 | 60 | func (p *PositionPlugin) Update(v string) error { 61 | // only updating memory variables is not persistent 62 | // select save func persistent 63 | p.mu.Lock() 64 | defer p.mu.Unlock() 65 | if v == "" { 66 | return errors.Errorf("empty value") 67 | } 68 | p.pos = v 69 | return nil 70 | } 71 | 72 | func (p *PositionPlugin) Save() error { 73 | // persistent save pos 74 | p.mu.Lock() 75 | defer p.mu.Unlock() 76 | err := p.metaDb.Update(func(tx *bolt.Tx) error { 77 | b := tx.Bucket([]byte(p.bucketName)) 78 | if b == nil { 79 | return fmt.Errorf("bucket:%s does not exist", p.bucketName) 80 | } 81 | err := b.Put([]byte(p.name), []byte(p.pos)) 82 | return err 83 | }) 84 | return err 85 | } 86 | 87 | func (p *PositionPlugin) Get() string { 88 | p.mu.Lock() 89 | defer p.mu.Unlock() 90 | return p.pos 91 | } 92 | 93 | func (p *PositionPlugin) Close() { 94 | close(p.stop) 95 | <-p.done 96 | if p.metaDb != nil { 97 | err := p.metaDb.Close() 98 | if err != nil { 99 | log.Errorf("close metaDb conn failed: %s", err.Error()) 100 | } 101 | } 102 | } 103 | 104 | func (p *PositionPlugin) getPosFromMetaDb() string { 105 | var pos []byte 106 | err := p.metaDb.Update(func(tx *bolt.Tx) error { 107 | b, err := tx.CreateBucketIfNotExists([]byte(p.bucketName)) 108 | if err != nil { 109 | return err 110 | } 111 | pos = b.Get([]byte(p.name)) 112 | return nil 113 | }) 114 | if err != nil { 115 | log.Fatalf("from metaDb get position: %s", err.Error()) 116 | } 117 | return string(pos) 118 | } 119 | 120 | func (p *PositionPlugin) getPosFromConfig() string { 121 | if pos := p.MysqlConfig.Options.StartGtid; pos != "" { 122 | return fmt.Sprintf("%v", pos) 123 | } 124 | return "" 125 | } 126 | 127 | func (p *PositionPlugin) getPosFromSource() string { 128 | db, err := getConn(p.MysqlConfig) 129 | if err != nil { 130 | log.Fatalf("conn db failed, %s", err.Error()) 131 | return "" 132 | } 133 | defer closeConn(db) 134 | var gtidMode string 135 | err = db.QueryRow(`select @@GLOBAL.gtid_mode`).Scan(>idMode) 136 | if err != nil { 137 | log.Fatalf("query gtid_mode failed, %s", err.Error()) 138 | } 139 | if gtidMode != "ON" { 140 | log.Fatalf("gtid_mode is not enabled") 141 | } 142 | var nowGtid string 143 | err = db.QueryRow(`SELECT @@GLOBAL.gtid_executed`).Scan(&nowGtid) 144 | if err != nil { 145 | log.Fatalf("query now gitd value failed, %s", err.Error()) 146 | } 147 | return nowGtid 148 | } 149 | 150 | func (p *PositionPlugin) timerSave() { 151 | ticker := time.NewTicker(3 * time.Second) 152 | defer ticker.Stop() 153 | for { 154 | select { 155 | case <-ticker.C: 156 | if err := p.Save(); err != nil { 157 | log.Fatalf("timer save position failed: %s", err.Error()) 158 | } 159 | // log.Debugf("timer save position: %s", p.pos) 160 | case <-p.stop: 161 | if err := p.Save(); err != nil { 162 | log.Fatalf("timer save position failed: %s", err.Error()) 163 | } 164 | log.Infof("last save position: %v", p.pos) 165 | p.done <- struct{}{} 166 | return 167 | 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /inputs/mysql/mysql_replication.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/go-mysql-org/go-mysql/mysql" 7 | "github.com/google/uuid" 8 | "github.com/siddontang/go-log/log" 9 | "github.com/sqlpub/qin-cdc/config" 10 | "github.com/sqlpub/qin-cdc/core" 11 | "github.com/sqlpub/qin-cdc/metas" 12 | "regexp" 13 | ) 14 | import "github.com/go-mysql-org/go-mysql/replication" 15 | 16 | type BinlogTailer struct { 17 | *config.MysqlConfig 18 | syncer *replication.BinlogSyncer 19 | inputPlugin *InputPlugin 20 | Pos mysql.Position 21 | GSet mysql.GTIDSet 22 | } 23 | 24 | func (b *BinlogTailer) New(inputPlugin *InputPlugin) { 25 | cfg := replication.BinlogSyncerConfig{ 26 | ServerID: getServerId(inputPlugin.MysqlConfig), 27 | Flavor: PluginName, 28 | Host: inputPlugin.Host, 29 | Port: uint16(inputPlugin.Port), 30 | User: inputPlugin.UserName, 31 | Password: inputPlugin.Password, 32 | Charset: DefaultCharset, 33 | } 34 | b.syncer = replication.NewBinlogSyncer(cfg) 35 | b.inputPlugin = inputPlugin 36 | } 37 | 38 | func (b *BinlogTailer) Start(gtid string) { 39 | gtidSet, err := mysql.ParseGTIDSet(PluginName, gtid) 40 | if err != nil { 41 | log.Fatalf("parse gtid %s with flavor %s failed, error: %v", gtid, PluginName, err.Error()) 42 | } 43 | streamer, _ := b.syncer.StartSyncGTID(gtidSet) 44 | for { 45 | ev, err := streamer.GetEvent(context.Background()) 46 | if err == replication.ErrSyncClosed { 47 | return 48 | } 49 | // ev.Dump(os.Stdout) 50 | switch e := ev.Event.(type) { 51 | case *replication.RotateEvent: 52 | b.handleRotateEvent(e) 53 | case *replication.RowsEvent: 54 | b.handleRowsEvent(ev) 55 | case *replication.XIDEvent: 56 | b.handleXIDEvent(ev) 57 | case *replication.GTIDEvent: 58 | b.handleGTIDEvent(e) 59 | case *replication.QueryEvent: 60 | b.handleDDLEvent(ev) 61 | default: 62 | continue 63 | } 64 | } 65 | } 66 | 67 | func (b *BinlogTailer) Close() { 68 | b.syncer.Close() 69 | } 70 | 71 | func (b *BinlogTailer) columnCountEqual(e *replication.RowsEvent, tableMeta *metas.Table) bool { 72 | return int(e.ColumnCount) == len(tableMeta.Columns) 73 | } 74 | 75 | func (b *BinlogTailer) handleRotateEvent(e *replication.RotateEvent) { 76 | b.Pos.Name = string(e.NextLogName) 77 | b.Pos.Pos = uint32(e.Position) 78 | } 79 | 80 | func (b *BinlogTailer) handleRowsEvent(ev *replication.BinlogEvent) { 81 | e := ev.Event.(*replication.RowsEvent) 82 | schemaName, tableName := string(e.Table.Schema), string(e.Table.Table) 83 | tableMeta, _ := b.inputPlugin.metaPlugin.Get(schemaName, tableName) 84 | if tableMeta == nil { 85 | return 86 | } 87 | // column count equal check 88 | if !b.columnCountEqual(e, tableMeta) { 89 | log.Fatalf("binlog event row values length: %d != table meta columns length: %d", e.ColumnCount, len(tableMeta.Columns)) 90 | } 91 | 92 | var msgs []*core.Msg 93 | var err error 94 | var actionType core.ActionType 95 | switch ev.Header.EventType { 96 | case replication.WRITE_ROWS_EVENTv0, replication.WRITE_ROWS_EVENTv1, replication.WRITE_ROWS_EVENTv2: 97 | actionType = core.InsertAction 98 | msgs, err = b.inputPlugin.NewInsertMsgs(schemaName, tableName, tableMeta, e, ev.Header) 99 | case replication.UPDATE_ROWS_EVENTv0, replication.UPDATE_ROWS_EVENTv1, replication.UPDATE_ROWS_EVENTv2: 100 | actionType = core.UpdateAction 101 | msgs, err = b.inputPlugin.NewUpdateMsgs(schemaName, tableName, tableMeta, e, ev.Header) 102 | case replication.DELETE_ROWS_EVENTv0, replication.DELETE_ROWS_EVENTv1, replication.DELETE_ROWS_EVENTv2: 103 | actionType = core.DeleteAction 104 | msgs, err = b.inputPlugin.NewDeleteMsgs(schemaName, tableName, tableMeta, e, ev.Header) 105 | default: 106 | log.Fatalf("unsupported event type: %s", ev.Header.EventType) 107 | } 108 | if err != nil { 109 | log.Fatalf("%v event handle failed: %s", actionType, err.Error()) 110 | } 111 | b.inputPlugin.SendMsgs(msgs) 112 | } 113 | 114 | func (b *BinlogTailer) handleXIDEvent(ev *replication.BinlogEvent) { 115 | e := ev.Event.(*replication.XIDEvent) 116 | msg, err := b.inputPlugin.NewXIDMsg(e, ev.Header) 117 | if err != nil { 118 | log.Fatalf("xid event handle failed: %s", err.Error()) 119 | } 120 | b.inputPlugin.SendMsg(msg) 121 | } 122 | 123 | func (b *BinlogTailer) handleGTIDEvent(e *replication.GTIDEvent) { 124 | var err error 125 | u, _ := uuid.FromBytes(e.SID) 126 | b.GSet, err = mysql.ParseMysqlGTIDSet(fmt.Sprintf("%s:%d", u.String(), e.GNO)) 127 | if err != nil { 128 | log.Fatalf("gtid event handle failed: %v", err) 129 | } 130 | } 131 | 132 | func (b *BinlogTailer) handleDDLEvent(ev *replication.BinlogEvent) { 133 | e := ev.Event.(*replication.QueryEvent) 134 | schemaName := string(e.Schema) 135 | ddlSql := string(e.Query) 136 | if ddlSql == "BEGIN" { 137 | return 138 | } 139 | ddlStatements, err := metas.TableDdlParser(ddlSql, schemaName) 140 | if err != nil { 141 | log.Fatalf("ddl event handle failed: %s", err.Error()) 142 | } 143 | for _, ddlStatement := range ddlStatements { 144 | schema := ddlStatement.Schema 145 | name := ddlStatement.Name 146 | var table *metas.Table 147 | table, err = b.inputPlugin.metaPlugin.Get(schema, name) 148 | if err != nil { 149 | log.Fatalf("ddl event handle get table meta failed: %s", err.Error()) 150 | } 151 | 152 | isSyncTable := false 153 | isOnlineDdlTable := false 154 | for _, v := range b.inputPlugin.metaPlugin.GetAll() { 155 | if schema == v.Schema && name == v.Name { 156 | isSyncTable = true 157 | break 158 | } 159 | 160 | // aliyun dms online ddl reg 161 | aliyunDMSOnlineDdlRegStr := fmt.Sprintf("^tp_\\d+_(ogt|del|ogl)_%s$", v.Name) 162 | aliyunDMSOnlineDdlReg, err := regexp.Compile(aliyunDMSOnlineDdlRegStr) 163 | if err != nil { 164 | log.Fatalf("parse aliyun dms online ddl regexp err %v", err.Error()) 165 | } 166 | // aliyun dms online ddl reg2 167 | aliyunDMSOnlineDdlReg2Str := fmt.Sprintf("^tpa_[a-z0-9]+_%v$", v.Name) 168 | aliyunDMSOnlineDdlReg2, err := regexp.Compile(aliyunDMSOnlineDdlReg2Str) 169 | if err != nil { 170 | log.Fatalf("parse aliyun dms online ddl regexp err %v", err.Error()) 171 | } 172 | // gh-ost online ddl reg 173 | ghostOnlineDdlRegStr := fmt.Sprintf("^_%s_(gho|ghc|del)$", v.Name) 174 | ghostOnlineDdlReg, err := regexp.Compile(ghostOnlineDdlRegStr) 175 | if err != nil { 176 | log.Fatalf("parse gh-ost online ddl regexp err %v", err.Error()) 177 | } 178 | if schema == v.Schema && 179 | (aliyunDMSOnlineDdlReg.MatchString(name) || 180 | aliyunDMSOnlineDdlReg2.MatchString(name) || 181 | ghostOnlineDdlReg.MatchString(name)) { 182 | isOnlineDdlTable = true 183 | break 184 | } 185 | } 186 | if isSyncTable || isOnlineDdlTable { 187 | deepCopy, err := table.DeepCopy() 188 | if err != nil { 189 | log.Fatalf("table deep copy failed: %s", err.Error()) 190 | } 191 | if ddlStatement.IsAlterTable { // alter table 192 | err = metas.TableDdlHandle(deepCopy, ddlStatement.RawSql) 193 | if err != nil { 194 | log.Fatalf("ddl event handle failed: %s", err.Error()) 195 | } 196 | err = b.inputPlugin.metaPlugin.Update(deepCopy) 197 | if err != nil { 198 | log.Fatalf("ddl event handle failed: %s", err.Error()) 199 | } 200 | } else if ddlStatement.IsCreateTable { 201 | err = metas.TableDdlHandle(deepCopy, ddlStatement.RawSql) 202 | if err != nil { 203 | log.Fatalf("ddl event handle failed: %s", err.Error()) 204 | } 205 | err = b.inputPlugin.metaPlugin.Add(deepCopy) 206 | if err != nil { 207 | log.Fatalf("ddl event handle failed: %s", err.Error()) 208 | } 209 | } else if ddlStatement.IsDropTable { // drop table 210 | err = b.inputPlugin.metaPlugin.Delete(schema, name) 211 | if err != nil { 212 | log.Fatalf("ddl event handle failed: %s", err.Error()) 213 | } 214 | } else if ddlStatement.IsRenameTable { // rename table 215 | err = metas.TableDdlHandle(deepCopy, ddlStatement.RawSql) 216 | if err != nil { 217 | log.Fatalf("ddl event handle failed: %s", err.Error()) 218 | } 219 | err = b.inputPlugin.metaPlugin.Update(deepCopy) 220 | if err != nil { 221 | log.Fatalf("ddl event handle failed: %s", err.Error()) 222 | } 223 | } else if ddlStatement.IsTruncateTable { // truncate table 224 | return 225 | } 226 | } 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /inputs/mysql/mysql_utils.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/siddontang/go-log/log" 7 | "github.com/sqlpub/qin-cdc/config" 8 | "github.com/sqlpub/qin-cdc/metas" 9 | "math/rand" 10 | "time" 11 | ) 12 | 13 | const ( 14 | PluginName = "mysql" 15 | DefaultCharset string = "utf8" 16 | ) 17 | 18 | func getConn(conf *config.MysqlConfig) (db *sql.DB, err error) { 19 | dsn := fmt.Sprintf( 20 | "%s:%s@tcp(%s:%d)/information_schema?charset=utf8mb4&timeout=3s", 21 | conf.UserName, conf.Password, 22 | conf.Host, conf.Port) 23 | db, err = sql.Open("mysql", dsn) 24 | if err != nil { 25 | return db, err 26 | } 27 | db.SetConnMaxLifetime(time.Minute * 3) 28 | db.SetMaxOpenConns(2) 29 | db.SetMaxIdleConns(2) 30 | return db, err 31 | } 32 | 33 | func closeConn(db *sql.DB) { 34 | if db != nil { 35 | _ = db.Close() 36 | } 37 | } 38 | 39 | func getServerId(conf *config.MysqlConfig) uint32 { 40 | serverId := conf.Options.ServerId 41 | if serverId > 0 { 42 | return uint32(serverId) 43 | } else if serverId < 0 { 44 | log.Fatalf("options server-id: %v is invalid, must be greater than 0", serverId) 45 | } 46 | // default generate rand value [1001, 2000] 47 | return uint32(rand.New(rand.NewSource(time.Now().Unix())).Intn(1000)) + 1001 48 | } 49 | 50 | func deserialize(raw interface{}, column metas.Column) interface{} { 51 | if raw == nil { 52 | return nil 53 | } 54 | 55 | ret := raw 56 | if column.RawType == "text" || column.RawType == "json" { 57 | _, ok := raw.([]uint8) 58 | if ok { 59 | ret = string(raw.([]uint8)) 60 | } 61 | } 62 | return ret 63 | } 64 | -------------------------------------------------------------------------------- /metas/mysql_ddl_parse.go: -------------------------------------------------------------------------------- 1 | package metas 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "github.com/pingcap/tidb/pkg/parser" 8 | "github.com/pingcap/tidb/pkg/parser/ast" 9 | "github.com/pingcap/tidb/pkg/parser/format" 10 | "github.com/pingcap/tidb/pkg/parser/mysql" 11 | "github.com/pingcap/tidb/pkg/parser/test_driver" 12 | ) 13 | 14 | var p *parser.Parser 15 | 16 | func init() { 17 | p = parser.New() 18 | } 19 | 20 | func parse(sql string) (*ast.StmtNode, error) { 21 | stmtNodes, _, err := p.ParseSQL(sql) 22 | if err != nil { 23 | return nil, err 24 | } 25 | 26 | return &stmtNodes[0], nil 27 | } 28 | 29 | func columnDefParse(columnDef *ast.ColumnDef) Column { 30 | tableColumn := Column{} 31 | tableColumn.Name = columnDef.Name.String() 32 | tableColumn.RawType = columnDef.Tp.String() 33 | switch columnDef.Tp.GetType() { 34 | case mysql.TypeEnum: 35 | tableColumn.Type = TypeEnum 36 | case mysql.TypeSet: 37 | tableColumn.Type = TypeSet 38 | case mysql.TypeTimestamp: 39 | tableColumn.Type = TypeTimestamp 40 | case mysql.TypeDatetime: 41 | tableColumn.Type = TypeDatetime 42 | case mysql.TypeDuration: 43 | tableColumn.Type = TypeTime 44 | case mysql.TypeDouble, mysql.TypeFloat: 45 | tableColumn.Type = TypeFloat 46 | case mysql.TypeNewDecimal: 47 | tableColumn.Type = TypeDecimal 48 | case mysql.TypeBit: 49 | tableColumn.Type = TypeBit 50 | case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString: 51 | tableColumn.Type = TypeString 52 | case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong, mysql.TypeLonglong, mysql.TypeYear: 53 | tableColumn.Type = TypeNumber 54 | case mysql.TypeDate: 55 | tableColumn.Type = TypeDate 56 | case mysql.TypeJSON: 57 | tableColumn.Type = TypeJson 58 | case mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeBlob, mysql.TypeLongBlob: 59 | tableColumn.Type = TypeBinary 60 | } 61 | for _, columnOption := range columnDef.Options { 62 | switch columnOption.Tp { 63 | case ast.ColumnOptionNoOption: 64 | case ast.ColumnOptionPrimaryKey: 65 | tableColumn.IsPrimaryKey = true 66 | case ast.ColumnOptionNotNull: 67 | case ast.ColumnOptionAutoIncrement: 68 | case ast.ColumnOptionDefaultValue: 69 | case ast.ColumnOptionUniqKey: 70 | case ast.ColumnOptionNull: 71 | case ast.ColumnOptionOnUpdate: // For Timestamp and Datetime only. 72 | case ast.ColumnOptionFulltext: 73 | case ast.ColumnOptionComment: 74 | switch exp := columnOption.Expr.(type) { 75 | case *test_driver.ValueExpr: 76 | tableColumn.Comment = exp.Datum.GetString() 77 | } 78 | case ast.ColumnOptionGenerated: 79 | case ast.ColumnOptionReference: 80 | case ast.ColumnOptionCollate: 81 | case ast.ColumnOptionCheck: 82 | case ast.ColumnOptionColumnFormat: 83 | case ast.ColumnOptionStorage: 84 | case ast.ColumnOptionAutoRandom: 85 | } 86 | } 87 | return tableColumn 88 | } 89 | 90 | func NewTable(createDdlSql string) (*Table, error) { 91 | tab := &Table{} 92 | err := TableDdlHandle(tab, createDdlSql) 93 | if err != nil { 94 | return nil, err 95 | } 96 | return tab, nil 97 | } 98 | 99 | func TableDdlHandle(tab *Table, sql string) error { 100 | astNode, err := parse(sql) 101 | if err != nil { 102 | return errors.New(fmt.Sprintf("parse error: %v\n", err.Error())) 103 | } 104 | switch t := (*astNode).(type) { 105 | case *ast.AlterTableStmt: 106 | if t.Table.Schema.String() != tab.Schema || t.Table.Name.String() != tab.Name { 107 | return errors.New(fmt.Sprintf("operation object do not match error: table: %s.%s and sql: %s", tab.Schema, tab.Name, sql)) 108 | } 109 | for _, alterTableSpec := range t.Specs { 110 | switch alterTableSpec.Tp { 111 | case ast.AlterTableOption: 112 | case ast.AlterTableAddColumns: 113 | relativeColumn := "" 114 | isFirst := false 115 | switch alterTableSpec.Position.Tp { 116 | case ast.ColumnPositionNone: 117 | case ast.ColumnPositionFirst: 118 | isFirst = true 119 | case ast.ColumnPositionAfter: 120 | relativeColumn = alterTableSpec.Position.RelativeColumn.Name.String() 121 | } 122 | for _, column := range alterTableSpec.NewColumns { 123 | tableColumn := columnDefParse(column) 124 | if tableColumn.IsPrimaryKey { 125 | tab.PrimaryKeyColumns = append(tab.PrimaryKeyColumns, tableColumn) 126 | } 127 | if relativeColumn != "" { 128 | for i, column2 := range tab.Columns { 129 | // add new column to relative column after 130 | if column2.Name == relativeColumn { 131 | tab.Columns = append(tab.Columns[:i+1], append([]Column{tableColumn}, tab.Columns[i+1:]...)...) 132 | } 133 | } 134 | } else if isFirst { 135 | // add new column to first 136 | tab.Columns = append([]Column{tableColumn}, tab.Columns...) 137 | } else { 138 | tab.Columns = append(tab.Columns, tableColumn) 139 | } 140 | } 141 | case ast.AlterTableAddConstraint: 142 | case ast.AlterTableDropColumn: 143 | oldColumnName := alterTableSpec.OldColumnName.Name.String() 144 | for i, column := range tab.Columns { 145 | if column.Name == oldColumnName { 146 | tab.Columns = append(tab.Columns[:i], tab.Columns[i+1:]...) 147 | } 148 | } 149 | case ast.AlterTableDropPrimaryKey: 150 | case ast.AlterTableDropIndex: 151 | case ast.AlterTableDropForeignKey: 152 | case ast.AlterTableModifyColumn: 153 | relativeColumn := "" 154 | isFirst := false 155 | switch alterTableSpec.Position.Tp { 156 | case ast.ColumnPositionNone: 157 | case ast.ColumnPositionFirst: 158 | isFirst = true 159 | case ast.ColumnPositionAfter: 160 | relativeColumn = alterTableSpec.Position.RelativeColumn.Name.String() 161 | } 162 | for _, column := range alterTableSpec.NewColumns { 163 | tableColumn := columnDefParse(column) 164 | if tableColumn.IsPrimaryKey { 165 | tab.PrimaryKeyColumns = append(tab.PrimaryKeyColumns, tableColumn) 166 | } 167 | if relativeColumn != "" { 168 | for i, column2 := range tab.Columns { 169 | // delete old column 170 | if column2.Name == tableColumn.Name { 171 | tab.Columns = append(tab.Columns[:i], tab.Columns[i+1:]...) 172 | } 173 | } 174 | for i, column2 := range tab.Columns { 175 | // add new column to relative column after 176 | if column2.Name == relativeColumn { 177 | tab.Columns = append(tab.Columns[:i+1], append([]Column{tableColumn}, tab.Columns[i+1:]...)...) 178 | } 179 | } 180 | } else if isFirst { 181 | for i, column2 := range tab.Columns { 182 | // delete old column 183 | if column2.Name == tableColumn.Name { 184 | tab.Columns = append(tab.Columns[:i], tab.Columns[i+1:]...) 185 | } 186 | } 187 | // add new column to first 188 | tab.Columns = append([]Column{tableColumn}, tab.Columns...) 189 | } else { 190 | // overwrite column 191 | for i, column2 := range tab.Columns { 192 | if column2.Name == tableColumn.Name { 193 | tab.Columns[i] = tableColumn 194 | break 195 | } 196 | } 197 | } 198 | } 199 | case ast.AlterTableChangeColumn: 200 | oldColumnName := alterTableSpec.OldColumnName.Name.String() 201 | relativeColumn := "" 202 | isFirst := false 203 | switch alterTableSpec.Position.Tp { 204 | case ast.ColumnPositionNone: 205 | case ast.ColumnPositionFirst: 206 | isFirst = true 207 | case ast.ColumnPositionAfter: 208 | relativeColumn = alterTableSpec.Position.RelativeColumn.Name.String() 209 | } 210 | for _, column := range alterTableSpec.NewColumns { 211 | tableColumn := columnDefParse(column) 212 | if tableColumn.IsPrimaryKey { 213 | tab.PrimaryKeyColumns = append(tab.PrimaryKeyColumns, tableColumn) 214 | } 215 | if relativeColumn != "" { 216 | for i, column2 := range tab.Columns { 217 | // delete old column 218 | if column2.Name == oldColumnName { 219 | tab.Columns = append(tab.Columns[:i], tab.Columns[i+1:]...) 220 | } 221 | } 222 | for i, column2 := range tab.Columns { 223 | // add new column to relative column after 224 | if column2.Name == relativeColumn { 225 | tab.Columns = append(tab.Columns[:i+1], append([]Column{tableColumn}, tab.Columns[i+1:]...)...) 226 | } 227 | } 228 | } else if isFirst { 229 | for i, column2 := range tab.Columns { 230 | // delete old column 231 | if column2.Name == oldColumnName { 232 | tab.Columns = append(tab.Columns[:i], tab.Columns[i+1:]...) 233 | } 234 | } 235 | // add new column to first 236 | tab.Columns = append([]Column{tableColumn}, tab.Columns...) 237 | } else { 238 | // overwrite column 239 | for i, column2 := range tab.Columns { 240 | if column2.Name == oldColumnName { 241 | tab.Columns[i] = tableColumn 242 | break 243 | } 244 | } 245 | } 246 | } 247 | case ast.AlterTableRenameColumn: 248 | oldColumnName := alterTableSpec.OldColumnName.Name.String() 249 | newColumnName := alterTableSpec.NewColumnName.Name.String() 250 | for i, column := range tab.Columns { 251 | if column.Name == oldColumnName { 252 | tab.Columns[i].Name = newColumnName 253 | break 254 | } 255 | } 256 | case ast.AlterTableRenameTable: 257 | newTableName := alterTableSpec.NewTable.Name.String() 258 | newSchemaName := alterTableSpec.NewTable.Schema.String() 259 | tab.Name = newTableName 260 | if newSchemaName != "" { 261 | tab.Schema = newSchemaName 262 | } 263 | case ast.AlterTableAlterColumn: 264 | case ast.AlterTableLock: 265 | case ast.AlterTableWriteable: 266 | case ast.AlterTableAlgorithm: 267 | case ast.AlterTableRenameIndex: 268 | case ast.AlterTableForce: 269 | case ast.AlterTableAddPartitions: 270 | case ast.AlterTablePartitionAttributes: 271 | case ast.AlterTablePartitionOptions: 272 | case ast.AlterTableCoalescePartitions: 273 | case ast.AlterTableDropPartition: 274 | case ast.AlterTableTruncatePartition: 275 | case ast.AlterTablePartition: 276 | case ast.AlterTableEnableKeys: 277 | case ast.AlterTableDisableKeys: 278 | case ast.AlterTableRemovePartitioning: 279 | case ast.AlterTableWithValidation: 280 | case ast.AlterTableWithoutValidation: 281 | case ast.AlterTableSecondaryLoad: 282 | case ast.AlterTableSecondaryUnload: 283 | case ast.AlterTableRebuildPartition: 284 | case ast.AlterTableReorganizePartition: 285 | case ast.AlterTableCheckPartitions: 286 | case ast.AlterTableExchangePartition: 287 | case ast.AlterTableOptimizePartition: 288 | case ast.AlterTableRepairPartition: 289 | case ast.AlterTableImportPartitionTablespace: 290 | case ast.AlterTableDiscardPartitionTablespace: 291 | case ast.AlterTableAlterCheck: 292 | case ast.AlterTableDropCheck: 293 | case ast.AlterTableImportTablespace: 294 | case ast.AlterTableDiscardTablespace: 295 | case ast.AlterTableIndexInvisible: 296 | case ast.AlterTableOrderByColumns: 297 | case ast.AlterTableSetTiFlashReplica: 298 | case ast.AlterTableAddStatistics: 299 | case ast.AlterTableDropStatistics: 300 | case ast.AlterTableAttributes: 301 | case ast.AlterTableCache: 302 | case ast.AlterTableNoCache: 303 | case ast.AlterTableStatsOptions: 304 | case ast.AlterTableDropFirstPartition: 305 | case ast.AlterTableAddLastPartition: 306 | case ast.AlterTableReorganizeLastPartition: 307 | case ast.AlterTableReorganizeFirstPartition: 308 | case ast.AlterTableRemoveTTL: 309 | } 310 | } 311 | case *ast.CreateTableStmt: 312 | if tab.Schema != "" || tab.Name != "" { 313 | return errors.New("table object not empty error: create table sql table object must be empty") 314 | } 315 | tab.Schema = t.Table.Schema.String() 316 | tab.Name = t.Table.Name.String() 317 | for _, tableOption := range t.Options { 318 | switch tableOption.Tp { 319 | case ast.TableOptionComment: 320 | tab.Comment = tableOption.StrValue 321 | default: 322 | continue 323 | } 324 | } 325 | tab.Columns = []Column{} 326 | for _, columnDef := range t.Cols { 327 | tableColumn := columnDefParse(columnDef) 328 | if tableColumn.IsPrimaryKey { 329 | tab.PrimaryKeyColumns = append(tab.PrimaryKeyColumns, tableColumn) 330 | } 331 | tab.Columns = append(tab.Columns, tableColumn) 332 | } 333 | for _, constraint := range t.Constraints { 334 | switch constraint.Tp { 335 | case ast.ConstraintPrimaryKey: 336 | for _, key := range constraint.Keys { 337 | keyName := key.Column.Name.String() 338 | for i, column := range tab.Columns { 339 | if keyName == column.Name { 340 | tab.Columns[i].IsPrimaryKey = true 341 | tab.PrimaryKeyColumns = append(tab.PrimaryKeyColumns, tab.Columns[i]) 342 | break 343 | } 344 | } 345 | } 346 | case ast.ConstraintKey: 347 | case ast.ConstraintIndex: 348 | case ast.ConstraintUniq: 349 | case ast.ConstraintUniqKey: 350 | case ast.ConstraintUniqIndex: 351 | case ast.ConstraintForeignKey: 352 | case ast.ConstraintFulltext: 353 | case ast.ConstraintCheck: 354 | default: 355 | } 356 | } 357 | case *ast.RenameTableStmt: 358 | for _, tableToTable := range t.TableToTables { 359 | if tableToTable.OldTable.Schema.String() != tab.Schema || tableToTable.OldTable.Name.String() != tab.Name { 360 | return errors.New(fmt.Sprintf("operation object do not match error: table: %s.%s and sql: %s", tab.Schema, tab.Name, sql)) 361 | } 362 | newSchema := tableToTable.NewTable.Schema.String() 363 | newTable := tableToTable.NewTable.Name.String() 364 | tab.Schema = newSchema 365 | tab.Name = newTable 366 | break 367 | } 368 | case *ast.DropTableStmt: 369 | return nil 370 | case *ast.TruncateTableStmt: 371 | return nil 372 | default: 373 | return errors.New(fmt.Sprintf("not support table ddl handle, type: %v", t)) 374 | } 375 | return nil 376 | } 377 | 378 | func TableDdlParser(sql string, schema string) ([]*DdlStatement, error) { 379 | astNode, err := parse(sql) 380 | if err != nil { 381 | return nil, errors.New(fmt.Sprintf("parse error: %v\n", err.Error())) 382 | } 383 | ddls := make([]*DdlStatement, 0) 384 | switch t := (*astNode).(type) { 385 | case *ast.AlterTableStmt: 386 | ddl := &DdlStatement{} 387 | tableSchema := t.Table.Schema.String() 388 | if tableSchema != "" { 389 | ddl.Schema = tableSchema 390 | } else { 391 | ddl.Schema = schema 392 | t.Table.Schema.L = schema 393 | t.Table.Schema.O = schema 394 | } 395 | ddl.Name = t.Table.Name.String() 396 | ddl.RawSql, err = TableRestore(t) 397 | if err != nil { 398 | return nil, err 399 | } 400 | ddl.IsAlterTable = true 401 | ddls = append(ddls, ddl) 402 | case *ast.CreateTableStmt: 403 | ddl := &DdlStatement{} 404 | tableSchema := t.Table.Schema.String() 405 | if tableSchema != "" { 406 | ddl.Schema = tableSchema 407 | } else { 408 | ddl.Schema = schema 409 | t.Table.Schema.L = schema 410 | t.Table.Schema.O = schema 411 | } 412 | ddl.Name = t.Table.Name.String() 413 | // CREATE TABLE ... LIKE Statement 414 | if t.ReferTable != nil { 415 | referTableSchema := t.ReferTable.Schema.String() 416 | if referTableSchema != "" { 417 | ddl.CreateTable.ReferTable.Schema = referTableSchema 418 | } else { 419 | ddl.CreateTable.ReferTable.Schema = schema 420 | t.ReferTable.Schema.L = schema 421 | t.ReferTable.Schema.O = schema 422 | } 423 | ddl.CreateTable.ReferTable.Name = t.ReferTable.Name.String() 424 | ddl.CreateTable.IsLikeCreateTable = true 425 | } 426 | // CREATE TABLE ... SELECT Statement 427 | if t.Select != nil { 428 | ddl.CreateTable.IsSelectCreateTable = true 429 | ddl.RawSql, err = TableRestore(t.Select) 430 | if err != nil { 431 | return nil, err 432 | } 433 | } 434 | ddl.IsCreateTable = true 435 | ddl.RawSql, err = TableRestore(astNode) 436 | if err != nil { 437 | return nil, err 438 | } 439 | ddls = append(ddls, ddl) 440 | case *ast.DropTableStmt: 441 | for _, tableName := range t.Tables { 442 | ddl := &DdlStatement{} 443 | tableSchema := tableName.Schema.String() 444 | if tableSchema != "" { 445 | ddl.Schema = tableSchema 446 | } else { 447 | ddl.Schema = schema 448 | tableName.Schema.L = schema 449 | tableName.Schema.O = schema 450 | } 451 | ddl.Name = tableName.Name.String() 452 | ddl.RawSql, err = TableRestore(tableName) 453 | if err != nil { 454 | return nil, err 455 | } 456 | ddl.IsDropTable = true 457 | ddls = append(ddls, ddl) 458 | } 459 | case *ast.RenameTableStmt: 460 | for _, tableToTable := range t.TableToTables { 461 | ddl := &DdlStatement{} 462 | oldSchema := tableToTable.OldTable.Schema.String() 463 | if oldSchema != "" { 464 | ddl.Schema = oldSchema 465 | } else { 466 | ddl.Schema = schema 467 | tableToTable.OldTable.Schema.L = schema 468 | tableToTable.OldTable.Schema.O = schema 469 | } 470 | ddl.Name = tableToTable.OldTable.Name.String() 471 | 472 | newSchema := tableToTable.NewTable.Schema.String() 473 | if newSchema == "" { 474 | tableToTable.NewTable.Schema.L = schema 475 | tableToTable.NewTable.Schema.O = schema 476 | } 477 | ddl.RawSql, err = TableRestore(tableToTable) 478 | if err != nil { 479 | return nil, err 480 | } 481 | ddl.IsRenameTable = true 482 | ddls = append(ddls, ddl) 483 | } 484 | case *ast.TruncateTableStmt: 485 | ddl := &DdlStatement{} 486 | tableSchema := t.Table.Schema.String() 487 | if tableSchema != "" { 488 | ddl.Schema = tableSchema 489 | } else { 490 | ddl.Schema = schema 491 | t.Table.Schema.L = schema 492 | t.Table.Schema.O = schema 493 | } 494 | ddl.Name = t.Table.Name.String() 495 | ddl.RawSql, err = TableRestore(t) 496 | if err != nil { 497 | return nil, err 498 | } 499 | ddl.IsTruncateTable = true 500 | ddls = append(ddls, ddl) 501 | default: 502 | return nil, errors.New(fmt.Sprintf("not support table ddl parser, type: %v", t)) 503 | } 504 | return ddls, nil 505 | } 506 | 507 | func TableRestore(astNode interface{}) (rawSql string, err error) { 508 | switch t := astNode.(type) { 509 | case *ast.AlterTableStmt: 510 | buf := new(bytes.Buffer) 511 | restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, buf) 512 | err = t.Restore(restoreCtx) 513 | if err != nil { 514 | return "", errors.New(fmt.Sprintf("table restore parse error: %v", err.Error())) 515 | } 516 | return buf.String(), nil 517 | case *ast.CreateTableStmt: 518 | buf := new(bytes.Buffer) 519 | restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, buf) 520 | err = t.Restore(restoreCtx) 521 | if err != nil { 522 | return "", errors.New(fmt.Sprintf("table restore parse error: %v", err.Error())) 523 | } 524 | return buf.String(), nil 525 | case *ast.TruncateTableStmt: 526 | buf := new(bytes.Buffer) 527 | restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, buf) 528 | err = t.Restore(restoreCtx) 529 | if err != nil { 530 | return "", errors.New(fmt.Sprintf("table restore parse error: %v", err.Error())) 531 | } 532 | return buf.String(), nil 533 | case ast.ResultSetNode: // CREATE TABLE ... SELECT Statement 534 | buf := new(bytes.Buffer) 535 | restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, buf) 536 | err = t.Restore(restoreCtx) 537 | if err != nil { 538 | return "", errors.New(fmt.Sprintf("table restore parse error: %v", err.Error())) 539 | } 540 | return buf.String(), nil 541 | case *ast.TableName: // DropTableStmt 542 | buf := new(bytes.Buffer) 543 | restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, buf) 544 | restoreCtx.WriteKeyWord("DROP TABLE ") 545 | err = t.Restore(restoreCtx) 546 | if err != nil { 547 | return "", errors.New(fmt.Sprintf("table restore parse error: %v", err.Error())) 548 | } 549 | return buf.String(), nil 550 | case *ast.TableToTable: // RenameTableStmt 551 | buf := new(bytes.Buffer) 552 | restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, buf) 553 | restoreCtx.WriteKeyWord("RENAME TABLE ") 554 | err = t.Restore(restoreCtx) 555 | if err != nil { 556 | return "", errors.New(fmt.Sprintf("table restore parse error: %v", err.Error())) 557 | } 558 | return buf.String(), nil 559 | default: 560 | return "", errors.New(fmt.Sprintf("not support table restore, type: %v", t)) 561 | } 562 | } 563 | -------------------------------------------------------------------------------- /metas/routers.go: -------------------------------------------------------------------------------- 1 | package metas 2 | 3 | import ( 4 | "github.com/mitchellh/mapstructure" 5 | "github.com/siddontang/go-log/log" 6 | "strconv" 7 | "strings" 8 | ) 9 | 10 | type Router struct { 11 | SourceSchema string `mapstructure:"source-schema"` 12 | SourceTable string `mapstructure:"source-table"` 13 | TargetSchema string `mapstructure:"target-schema"` 14 | TargetTable string `mapstructure:"target-table"` 15 | DmlTopic string `mapstructure:"dml-topic"` 16 | ColumnsMapper ColumnsMapper 17 | } 18 | 19 | type ColumnsMapper struct { 20 | PrimaryKeys []string // source 21 | SourceColumns []string 22 | TargetColumns []string 23 | MapMapper map[string]string 24 | MapMapperOrder []string 25 | } 26 | 27 | type Routers struct { 28 | Raws []*Router 29 | Maps map[string]*Router 30 | } 31 | 32 | var MapRouterKeyDelimiter = ":" 33 | 34 | func (r *Routers) InitRouters(config map[string]interface{}) error { 35 | r.initRaws(config) 36 | r.initMaps() 37 | return nil 38 | } 39 | 40 | func (r *Routers) initRaws(config map[string]interface{}) { 41 | configRules := config["routers"] 42 | err := mapstructure.Decode(configRules, &r.Raws) 43 | if err != nil { 44 | log.Fatal("output.config.routers config parsing failed. err: ", err.Error()) 45 | } 46 | } 47 | 48 | func (r *Routers) initMaps() { 49 | if len(r.Raws) == 0 { 50 | log.Fatal("routers config cannot be empty") 51 | } 52 | r.Maps = make(map[string]*Router) 53 | for _, router := range r.Raws { 54 | r.Maps[GenerateMapRouterKey(router.SourceSchema, router.SourceTable)] = router 55 | } 56 | } 57 | 58 | func GenerateMapRouterKey(schema string, table string) string { 59 | return schema + MapRouterKeyDelimiter + table 60 | } 61 | 62 | func GenerateMapRouterVersionKey(schema string, table string, version uint) string { 63 | return schema + MapRouterKeyDelimiter + table + MapRouterKeyDelimiter + strconv.Itoa(int(version)) 64 | } 65 | 66 | func SplitMapRouterKey(key string) (schema string, table string) { 67 | splits := strings.Split(key, MapRouterKeyDelimiter) 68 | return splits[0], splits[1] 69 | } 70 | 71 | func SplitMapRouterVersionKey(key string) (schema string, table string, version uint) { 72 | splits := strings.Split(key, MapRouterKeyDelimiter) 73 | tmpVersion, _ := strconv.Atoi(splits[2]) 74 | return splits[0], splits[1], uint(tmpVersion) 75 | } 76 | -------------------------------------------------------------------------------- /metas/table.go: -------------------------------------------------------------------------------- 1 | package metas 2 | 3 | import "github.com/goccy/go-json" 4 | 5 | type ColumnType = int 6 | 7 | const ( 8 | TypeNumber ColumnType = iota + 1 // tinyint, smallint, mediumint, int, bigint, year 9 | TypeFloat // float, double 10 | TypeEnum // enum 11 | TypeSet // set 12 | TypeString // other 13 | TypeDatetime // datetime 14 | TypeTimestamp // timestamp 15 | TypeDate // date 16 | TypeTime // time 17 | TypeBit // bit 18 | TypeJson // json 19 | TypeDecimal // decimal 20 | TypeBinary // binary 21 | ) 22 | 23 | type Table struct { 24 | Schema string 25 | Name string 26 | Comment string 27 | Columns []Column 28 | PrimaryKeyColumns []Column 29 | Version uint 30 | } 31 | 32 | type Column struct { 33 | Name string 34 | Type ColumnType 35 | RawType string 36 | Comment string 37 | IsPrimaryKey bool 38 | } 39 | 40 | type DdlStatement struct { 41 | Schema string 42 | Name string 43 | RawSql string 44 | IsAlterTable bool 45 | IsCreateTable bool 46 | CreateTable struct { 47 | IsLikeCreateTable bool 48 | ReferTable struct { 49 | Schema string 50 | Name string 51 | } 52 | IsSelectCreateTable bool 53 | SelectRawSql string 54 | } 55 | IsDropTable bool 56 | IsRenameTable bool 57 | IsTruncateTable bool 58 | } 59 | 60 | func (t *Table) DeepCopy() (*Table, error) { 61 | b, err := json.Marshal(t) 62 | if err != nil { 63 | return nil, err 64 | } 65 | ret := &Table{} 66 | if err = json.Unmarshal(b, ret); err != nil { 67 | panic(err) 68 | } 69 | return ret, nil 70 | } 71 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/prometheus/client_golang/prometheus" 5 | "github.com/prometheus/client_golang/prometheus/promauto" 6 | ) 7 | 8 | var OpsStartTime = prometheus.NewGauge(prometheus.GaugeOpts{ 9 | Namespace: "qin_cdc", 10 | Subsystem: "start", 11 | Name: "time", 12 | Help: "qin-cdc startup timestamp（s）.", 13 | }) 14 | 15 | var OpsReadProcessed = promauto.NewCounter(prometheus.CounterOpts{ 16 | Name: "qin_cdc_read_processed_ops_total", 17 | Help: "The total number of read processed events", 18 | }) 19 | 20 | var OpsWriteProcessed = promauto.NewCounter(prometheus.CounterOpts{ 21 | Name: "qin_cdc_write_processed_ops_total", 22 | Help: "The total number of write processed events", 23 | }) 24 | 25 | var DelayReadTime = prometheus.NewGauge(prometheus.GaugeOpts{ 26 | Namespace: "qin_cdc", 27 | Subsystem: "read_delay", 28 | Name: "time_seconds", 29 | Help: "Delay in seconds to read the binlog at the source.", 30 | }) 31 | 32 | var DelayWriteTime = prometheus.NewGauge(prometheus.GaugeOpts{ 33 | Namespace: "qin_cdc", 34 | Subsystem: "write_delay", 35 | Name: "time_seconds", 36 | Help: "Delay in seconds to write at the destination.", 37 | }) 38 | 39 | func init() { 40 | prometheus.MustRegister(OpsStartTime, DelayReadTime, DelayWriteTime) 41 | } 42 | -------------------------------------------------------------------------------- /outputs/doris/doris.go: -------------------------------------------------------------------------------- 1 | package doris 2 | 3 | import ( 4 | "fmt" 5 | "github.com/juju/errors" 6 | "github.com/mitchellh/mapstructure" 7 | "github.com/siddontang/go-log/log" 8 | "github.com/sqlpub/qin-cdc/config" 9 | "github.com/sqlpub/qin-cdc/core" 10 | "github.com/sqlpub/qin-cdc/metas" 11 | "github.com/sqlpub/qin-cdc/metrics" 12 | "io" 13 | "net/http" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | type OutputPlugin struct { 19 | *config.DorisConfig 20 | Done chan bool 21 | metas *core.Metas 22 | msgTxnBuffer struct { 23 | size int 24 | tableMsgMap map[string][]*core.Msg 25 | } 26 | client *http.Client 27 | transport *http.Transport 28 | lastPosition string 29 | } 30 | 31 | func (o *OutputPlugin) Configure(conf map[string]interface{}) error { 32 | o.DorisConfig = &config.DorisConfig{} 33 | var targetConf = conf["target"] 34 | if err := mapstructure.Decode(targetConf, o.DorisConfig); err != nil { 35 | return err 36 | } 37 | return nil 38 | } 39 | 40 | func (o *OutputPlugin) NewOutput(metas *core.Metas) { 41 | o.Done = make(chan bool) 42 | o.metas = metas 43 | // options handle 44 | if o.DorisConfig.Options.BatchSize == 0 { 45 | o.DorisConfig.Options.BatchSize = DefaultBatchSize 46 | } 47 | if o.DorisConfig.Options.BatchIntervalMs == 0 { 48 | o.DorisConfig.Options.BatchIntervalMs = DefaultBatchIntervalMs 49 | } 50 | o.msgTxnBuffer.size = 0 51 | o.msgTxnBuffer.tableMsgMap = make(map[string][]*core.Msg) 52 | 53 | o.transport = &http.Transport{} 54 | o.client = &http.Client{ 55 | Transport: o.transport, 56 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 57 | req.Header.Add("Authorization", "Basic "+o.auth()) 58 | // log.Debugf("重定向请求到be: %v", req.URL) 59 | return nil // return nil nil回重定向。 60 | }, 61 | } 62 | } 63 | 64 | func (o *OutputPlugin) Start(out chan *core.Msg, pos core.Position) { 65 | // first pos 66 | o.lastPosition = pos.Get() 67 | go func() { 68 | ticker := time.NewTicker(time.Millisecond * time.Duration(o.Options.BatchIntervalMs)) 69 | defer ticker.Stop() 70 | for { 71 | select { 72 | case data := <-out: 73 | switch data.Type { 74 | case core.MsgCtl: 75 | o.lastPosition = data.InputContext.Pos 76 | case core.MsgDML: 77 | o.appendMsgTxnBuffer(data) 78 | if o.msgTxnBuffer.size >= o.DorisConfig.Options.BatchSize { 79 | o.flushMsgTxnBuffer(pos) 80 | } 81 | } 82 | case <-ticker.C: 83 | o.flushMsgTxnBuffer(pos) 84 | case <-o.Done: 85 | o.flushMsgTxnBuffer(pos) 86 | return 87 | } 88 | 89 | } 90 | }() 91 | } 92 | 93 | func (o *OutputPlugin) Close() { 94 | log.Infof("output is closing...") 95 | close(o.Done) 96 | <-o.Done 97 | log.Infof("output is closed") 98 | } 99 | 100 | func (o *OutputPlugin) appendMsgTxnBuffer(msg *core.Msg) { 101 | key := metas.GenerateMapRouterKey(msg.Database, msg.Table) 102 | o.msgTxnBuffer.tableMsgMap[key] = append(o.msgTxnBuffer.tableMsgMap[key], msg) 103 | o.msgTxnBuffer.size += 1 104 | } 105 | 106 | func (o *OutputPlugin) flushMsgTxnBuffer(pos core.Position) { 107 | defer func() { 108 | // flush position 109 | err := pos.Update(o.lastPosition) 110 | if err != nil { 111 | log.Fatalf(err.Error()) 112 | } 113 | }() 114 | 115 | if o.msgTxnBuffer.size == 0 { 116 | return 117 | } 118 | // table level export 119 | for k, msgs := range o.msgTxnBuffer.tableMsgMap { 120 | columnsMapper := o.metas.Routers.Maps[k].ColumnsMapper 121 | targetSchema := o.metas.Routers.Maps[k].TargetSchema 122 | targetTable := o.metas.Routers.Maps[k].TargetTable 123 | err := o.execute(msgs, columnsMapper, targetSchema, targetTable) 124 | if err != nil { 125 | log.Fatalf("do %s bulk err %v", PluginName, err) 126 | } 127 | } 128 | o.clearMsgTxnBuffer() 129 | } 130 | 131 | func (o *OutputPlugin) clearMsgTxnBuffer() { 132 | o.msgTxnBuffer.size = 0 133 | o.msgTxnBuffer.tableMsgMap = make(map[string][]*core.Msg) 134 | } 135 | 136 | func (o *OutputPlugin) execute(msgs []*core.Msg, columnsMapper metas.ColumnsMapper, targetSchema string, targetTable string) error { 137 | if len(msgs) == 0 { 138 | return nil 139 | } 140 | var jsonList []string 141 | 142 | jsonList = o.generateJson(msgs) 143 | for _, s := range jsonList { 144 | log.Debugf("%s load %s.%s row data: %v", PluginName, targetSchema, targetTable, s) 145 | } 146 | log.Debugf("%s bulk load %s.%s row data num: %d", PluginName, targetSchema, targetTable, len(jsonList)) 147 | var err error 148 | for i := 0; i < RetryCount; i++ { 149 | err = o.sendData(jsonList, columnsMapper, targetSchema, targetTable) 150 | if err != nil { 151 | log.Warnf("send data failed, err: %v, execute retry...", err.Error()) 152 | if i+1 == RetryCount { 153 | break 154 | } 155 | time.Sleep(time.Duration(RetryInterval*(i+1)) * time.Second) 156 | continue 157 | } 158 | break 159 | } 160 | return err 161 | } 162 | 163 | func (o *OutputPlugin) sendData(content []string, columnsMapper metas.ColumnsMapper, targetSchema string, targetTable string) error { 164 | loadUrl := fmt.Sprintf("http://%s:%d/api/%s/%s/_stream_load", 165 | o.Host, o.LoadPort, targetSchema, targetTable) 166 | newContent := `[` + strings.Join(content, ",") + `]` 167 | req, _ := http.NewRequest("PUT", loadUrl, strings.NewReader(newContent)) 168 | 169 | // req.Header.Add 170 | req.Header.Add("Authorization", "Basic "+o.auth()) 171 | req.Header.Add("Expect", "100-continue") 172 | req.Header.Add("strict_mode", "true") 173 | // req.Header.Add("label", "39c25a5c-7000-496e-a98e-348a264c81de") 174 | req.Header.Add("format", "json") 175 | req.Header.Add("strip_outer_array", "true") 176 | req.Header.Add("merge_type", "MERGE") 177 | req.Header.Add("delete", DeleteCondition) 178 | 179 | var columnArray []string 180 | for _, column := range columnsMapper.SourceColumns { 181 | columnArray = append(columnArray, column) 182 | } 183 | columnArray = append(columnArray, DeleteColumn) 184 | columns := fmt.Sprintf("%s", strings.Join(columnArray, ",")) 185 | req.Header.Add("columns", columns) 186 | 187 | response, err := o.client.Do(req) 188 | if err != nil { 189 | return err 190 | } 191 | defer func(Body io.ReadCloser) { 192 | _ = Body.Close() 193 | }(response.Body) 194 | returnMap, err := o.parseResponse(response) 195 | if err != nil { 196 | return err 197 | } 198 | if returnMap["Status"] != "Success" { 199 | message := returnMap["Message"] 200 | errorUrl := returnMap["ErrorURL"] 201 | errorMsg := message.(string) + 202 | fmt.Sprintf(", targetTable: %s.%s", targetSchema, targetTable) + 203 | fmt.Sprintf(", visit ErrorURL to view error details, ErrorURL: %s", errorUrl) 204 | return errors.New(errorMsg) 205 | } 206 | // prom write event number counter 207 | numberLoadedRows := returnMap["NumberLoadedRows"] 208 | metrics.OpsWriteProcessed.Add(numberLoadedRows.(float64)) 209 | return nil 210 | } 211 | -------------------------------------------------------------------------------- /outputs/doris/doris_meta.go: -------------------------------------------------------------------------------- 1 | package doris 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | _ "github.com/go-sql-driver/mysql" 7 | "github.com/juju/errors" 8 | "github.com/mitchellh/mapstructure" 9 | "github.com/sqlpub/qin-cdc/config" 10 | "github.com/sqlpub/qin-cdc/metas" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | type MetaPlugin struct { 16 | *config.DorisConfig 17 | tables map[string]*metas.Table 18 | tablesVersion map[string]*metas.Table 19 | db *sql.DB 20 | mu sync.Mutex 21 | } 22 | 23 | func (m *MetaPlugin) Configure(conf map[string]interface{}) error { 24 | m.DorisConfig = &config.DorisConfig{} 25 | var target = conf["target"] 26 | if err := mapstructure.Decode(target, m.DorisConfig); err != nil { 27 | return err 28 | } 29 | return nil 30 | } 31 | 32 | func (m *MetaPlugin) LoadMeta(routers []*metas.Router) (err error) { 33 | m.tables = make(map[string]*metas.Table) 34 | m.tablesVersion = make(map[string]*metas.Table) 35 | dsn := fmt.Sprintf( 36 | "%s:%s@tcp(%s:%d)/information_schema?charset=utf8mb4&timeout=3s&interpolateParams=true", 37 | m.DorisConfig.UserName, m.DorisConfig.Password, 38 | m.DorisConfig.Host, m.DorisConfig.Port) 39 | m.db, err = sql.Open("mysql", dsn) 40 | if err != nil { 41 | return err 42 | } 43 | m.db.SetConnMaxLifetime(time.Minute * 3) 44 | m.db.SetMaxOpenConns(2) 45 | m.db.SetMaxIdleConns(2) 46 | err = m.db.Ping() 47 | if err != nil { 48 | return err 49 | } 50 | for _, router := range routers { 51 | rows, err := m.db.Query("select "+ 52 | "column_name,column_default,is_nullable,data_type,column_type,column_key "+ 53 | "from information_schema.columns "+ 54 | "where table_schema = ? and table_name = ? "+ 55 | "order by ordinal_position", router.TargetSchema, router.TargetTable) 56 | if err != nil { 57 | return err 58 | } 59 | table := &metas.Table{ 60 | Schema: router.TargetSchema, 61 | Name: router.TargetTable, 62 | } 63 | for rows.Next() { 64 | var columnName, isNullable, dataType, columnType, columnKey string 65 | var columnDefault sql.NullString 66 | err = rows.Scan(&columnName, &columnDefault, &isNullable, &dataType, &columnType, &columnKey) 67 | if err != nil { 68 | return err 69 | } 70 | var column metas.Column 71 | column.Name = columnName 72 | column.RawType = columnType 73 | switch dataType { 74 | case "tinyint", "smallint", "mediumint", "int", "bigint": 75 | column.Type = metas.TypeNumber 76 | case "float", "double": 77 | column.Type = metas.TypeFloat 78 | case "enum": 79 | column.Type = metas.TypeEnum 80 | case "set": 81 | column.Type = metas.TypeSet 82 | case "datetime": 83 | column.Type = metas.TypeDatetime 84 | case "timestamp": 85 | column.Type = metas.TypeTimestamp 86 | case "date": 87 | column.Type = metas.TypeDate 88 | case "time": 89 | column.Type = metas.TypeTime 90 | case "bit": 91 | column.Type = metas.TypeBit 92 | case "json": 93 | column.Type = metas.TypeJson 94 | case "decimal": 95 | column.Type = metas.TypeDecimal 96 | default: 97 | column.Type = metas.TypeString 98 | } 99 | if columnKey == "PRI" { 100 | column.IsPrimaryKey = true 101 | } 102 | table.Columns = append(table.Columns, column) 103 | } 104 | if table.Columns == nil { 105 | return errors.Errorf("load meta %s.%s not found", router.TargetSchema, router.TargetTable) 106 | } 107 | err = m.Add(table) 108 | if err != nil { 109 | return err 110 | } 111 | } 112 | return nil 113 | } 114 | 115 | func (m *MetaPlugin) GetMeta(router *metas.Router) (table interface{}, err error) { 116 | return m.Get(router.SourceSchema, router.SourceTable) 117 | } 118 | 119 | func (m *MetaPlugin) Get(schema string, tableName string) (table *metas.Table, err error) { 120 | key := metas.GenerateMapRouterKey(schema, tableName) 121 | m.mu.Lock() 122 | defer m.mu.Unlock() 123 | return m.tables[key], err 124 | } 125 | 126 | func (m *MetaPlugin) GetAll() map[string]*metas.Table { 127 | m.mu.Lock() 128 | defer m.mu.Unlock() 129 | return m.tables 130 | } 131 | 132 | func (m *MetaPlugin) GetVersion(schema string, tableName string, version uint) (table *metas.Table, err error) { 133 | key := metas.GenerateMapRouterVersionKey(schema, tableName, version) 134 | m.mu.Lock() 135 | defer m.mu.Unlock() 136 | return m.tablesVersion[key], err 137 | } 138 | 139 | func (m *MetaPlugin) GetVersions(schema string, tableName string) []*metas.Table { 140 | m.mu.Lock() 141 | defer m.mu.Unlock() 142 | tables := make([]*metas.Table, 0) 143 | for k, table := range m.tablesVersion { 144 | s, t, _ := metas.SplitMapRouterVersionKey(k) 145 | if schema == s && tableName == t { 146 | tables = append(tables, table) 147 | } 148 | } 149 | return tables 150 | } 151 | 152 | func (m *MetaPlugin) Add(newTable *metas.Table) error { 153 | m.mu.Lock() 154 | defer m.mu.Unlock() 155 | m.tables[metas.GenerateMapRouterKey(newTable.Schema, newTable.Name)] = newTable 156 | m.tablesVersion[metas.GenerateMapRouterVersionKey(newTable.Schema, newTable.Name, newTable.Version)] = newTable 157 | return nil 158 | } 159 | 160 | func (m *MetaPlugin) Update(newTable *metas.Table) error { 161 | m.mu.Lock() 162 | defer m.mu.Unlock() 163 | newTable.Version += 1 164 | m.tables[metas.GenerateMapRouterKey(newTable.Schema, newTable.Name)] = newTable 165 | m.tablesVersion[metas.GenerateMapRouterVersionKey(newTable.Schema, newTable.Name, newTable.Version)] = newTable 166 | return nil 167 | } 168 | 169 | func (m *MetaPlugin) Delete(schema string, name string) error { 170 | m.mu.Lock() 171 | defer m.mu.Unlock() 172 | delete(m.tables, metas.GenerateMapRouterKey(schema, name)) 173 | for _, table := range m.GetVersions(schema, name) { 174 | delete(m.tablesVersion, metas.GenerateMapRouterVersionKey(schema, name, table.Version)) 175 | } 176 | return nil 177 | } 178 | 179 | func (m *MetaPlugin) Save() error { 180 | return nil 181 | } 182 | 183 | func (m *MetaPlugin) Close() { 184 | if m.db != nil { 185 | _ = m.db.Close() 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /outputs/doris/doris_utils.go: -------------------------------------------------------------------------------- 1 | package doris 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "github.com/goccy/go-json" 7 | "github.com/siddontang/go-log/log" 8 | "github.com/sqlpub/qin-cdc/core" 9 | "io" 10 | "net/http" 11 | ) 12 | 13 | const ( 14 | PluginName = "doris" 15 | DefaultBatchSize int = 10240 16 | DefaultBatchIntervalMs int = 3000 17 | DeleteColumn string = "_delete_sign_" 18 | RetryCount int = 3 19 | RetryInterval int = 5 20 | ) 21 | 22 | var DeleteCondition = fmt.Sprintf("%s=1", DeleteColumn) 23 | 24 | func (o *OutputPlugin) auth() string { 25 | s := o.UserName + ":" + o.Password 26 | b := []byte(s) 27 | 28 | sEnc := base64.StdEncoding.EncodeToString(b) 29 | return sEnc 30 | } 31 | 32 | func (o *OutputPlugin) parseResponse(response *http.Response) (map[string]interface{}, error) { 33 | var result map[string]interface{} 34 | body, err := io.ReadAll(response.Body) 35 | if err == nil { 36 | err = json.Unmarshal(body, &result) 37 | } 38 | 39 | return result, err 40 | } 41 | 42 | func (o *OutputPlugin) generateJson(msgs []*core.Msg) []string { 43 | var jsonList []string 44 | 45 | for _, event := range msgs { 46 | switch event.DmlMsg.Action { 47 | case core.InsertAction: 48 | // 增加虚拟列，标识操作类型 (stream load opType：UPSERT 0，DELETE：1) 49 | event.DmlMsg.Data[DeleteColumn] = 0 50 | b, _ := json.Marshal(event.DmlMsg.Data) 51 | jsonList = append(jsonList, string(b)) 52 | case core.UpdateAction: 53 | // 增加虚拟列，标识操作类型 (stream load opType：UPSERT 0，DELETE：1) 54 | event.DmlMsg.Data[DeleteColumn] = 0 55 | b, _ := json.Marshal(event.DmlMsg.Data) 56 | jsonList = append(jsonList, string(b)) 57 | case core.DeleteAction: 58 | // 增加虚拟列，标识操作类型 (stream load opType：UPSERT 0，DELETE：1) 59 | event.DmlMsg.Data[DeleteColumn] = 1 60 | b, _ := json.Marshal(event.DmlMsg.Data) 61 | jsonList = append(jsonList, string(b)) 62 | case core.ReplaceAction: // for mongo 63 | // 增加虚拟列，标识操作类型 (stream load opType：UPSERT 0，DELETE：1) 64 | event.DmlMsg.Data[DeleteColumn] = 0 65 | b, _ := json.Marshal(event.DmlMsg.Data) 66 | jsonList = append(jsonList, string(b)) 67 | default: 68 | log.Fatalf("unhandled message type: %v", event) 69 | } 70 | } 71 | return jsonList 72 | } 73 | -------------------------------------------------------------------------------- /outputs/init.go: -------------------------------------------------------------------------------- 1 | package outputs 2 | 3 | import ( 4 | "github.com/sqlpub/qin-cdc/outputs/doris" 5 | "github.com/sqlpub/qin-cdc/outputs/kafka" 6 | "github.com/sqlpub/qin-cdc/outputs/mysql" 7 | "github.com/sqlpub/qin-cdc/outputs/starrocks" 8 | "github.com/sqlpub/qin-cdc/registry" 9 | ) 10 | 11 | func init() { 12 | // registry output plugins 13 | registry.RegisterPlugin(registry.OutputPlugin, starrocks.PluginName, &starrocks.OutputPlugin{}) 14 | registry.RegisterPlugin(registry.MetaPlugin, string(registry.OutputPlugin+starrocks.PluginName), &starrocks.MetaPlugin{}) 15 | 16 | registry.RegisterPlugin(registry.OutputPlugin, doris.PluginName, &doris.OutputPlugin{}) 17 | registry.RegisterPlugin(registry.MetaPlugin, string(registry.OutputPlugin+doris.PluginName), &doris.MetaPlugin{}) 18 | 19 | registry.RegisterPlugin(registry.OutputPlugin, mysql.PluginName, &mysql.OutputPlugin{}) 20 | registry.RegisterPlugin(registry.MetaPlugin, string(registry.OutputPlugin+mysql.PluginName), &mysql.MetaPlugin{}) 21 | 22 | registry.RegisterPlugin(registry.OutputPlugin, kafka.PluginName, &kafka.OutputPlugin{}) 23 | registry.RegisterPlugin(registry.MetaPlugin, string(registry.OutputPlugin+kafka.PluginName), &kafka.MetaPlugin{}) 24 | } 25 | -------------------------------------------------------------------------------- /outputs/kafka/kafka.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "fmt" 5 | gokafka "github.com/confluentinc/confluent-kafka-go/v2/kafka" 6 | "github.com/goccy/go-json" 7 | "github.com/mitchellh/mapstructure" 8 | "github.com/siddontang/go-log/log" 9 | "github.com/sqlpub/qin-cdc/config" 10 | "github.com/sqlpub/qin-cdc/core" 11 | "github.com/sqlpub/qin-cdc/metas" 12 | "github.com/sqlpub/qin-cdc/metrics" 13 | "strconv" 14 | "time" 15 | ) 16 | 17 | type OutputPlugin struct { 18 | *config.KafkaConfig 19 | Done chan bool 20 | metas *core.Metas 21 | formatInterface formatInterface 22 | msgTxnBuffer struct { 23 | size int 24 | tableMsgMap map[string][]*core.Msg 25 | } 26 | client *gokafka.Producer 27 | lastPosition string 28 | } 29 | 30 | func (o *OutputPlugin) Configure(conf map[string]interface{}) error { 31 | o.KafkaConfig = &config.KafkaConfig{} 32 | var targetConf = conf["target"] 33 | if err := mapstructure.Decode(targetConf, o.KafkaConfig); err != nil { 34 | return err 35 | } 36 | o.initFormatPlugin(fmt.Sprintf("%v", o.Options.OutputFormat)) 37 | return nil 38 | } 39 | 40 | func (o *OutputPlugin) NewOutput(metas *core.Metas) { 41 | o.Done = make(chan bool) 42 | o.metas = metas 43 | // options handle 44 | if o.KafkaConfig.Options.BatchSize == 0 { 45 | o.KafkaConfig.Options.BatchSize = DefaultBatchSize 46 | } 47 | if o.KafkaConfig.Options.BatchIntervalMs == 0 { 48 | o.KafkaConfig.Options.BatchIntervalMs = DefaultBatchIntervalMs 49 | } 50 | 51 | o.msgTxnBuffer.size = 0 52 | o.msgTxnBuffer.tableMsgMap = make(map[string][]*core.Msg) 53 | 54 | var err error 55 | o.client, err = getProducer(o.KafkaConfig) 56 | if err != nil { 57 | log.Fatal("output config client failed. err: ", err.Error()) 58 | } 59 | } 60 | 61 | func (o *OutputPlugin) Start(out chan *core.Msg, pos core.Position) { 62 | // first pos 63 | o.lastPosition = pos.Get() 64 | go func() { 65 | ticker := time.NewTicker(time.Millisecond * time.Duration(o.Options.BatchIntervalMs)) 66 | defer ticker.Stop() 67 | for { 68 | select { 69 | case data := <-out: 70 | switch data.Type { 71 | case core.MsgCtl: 72 | o.lastPosition = data.InputContext.Pos 73 | case core.MsgDML: 74 | o.appendMsgTxnBuffer(data) 75 | if o.msgTxnBuffer.size >= o.KafkaConfig.Options.BatchSize { 76 | o.flushMsgTxnBuffer(pos) 77 | } 78 | } 79 | case e := <-o.client.Events(): 80 | switch ev := e.(type) { 81 | case *gokafka.Message: 82 | m := ev 83 | if m.TopicPartition.Error != nil { 84 | log.Fatalf("Delivery failed: %v", m.TopicPartition.Error) 85 | } 86 | _, ok := m.Opaque.(*core.Msg) 87 | if !ok { 88 | log.Fatalf("kafka send failed to get meta data") 89 | } 90 | case gokafka.Error: 91 | log.Fatalf("kafka producer failed, err: %v", ev) 92 | default: 93 | log.Infof("Ignored event: %s", ev) 94 | } 95 | case <-ticker.C: 96 | o.flushMsgTxnBuffer(pos) 97 | case <-o.Done: 98 | o.flushMsgTxnBuffer(pos) 99 | return 100 | } 101 | 102 | } 103 | }() 104 | } 105 | 106 | func (o *OutputPlugin) Close() { 107 | log.Infof("output is closing...") 108 | close(o.Done) 109 | <-o.Done 110 | closeProducer(o.client) 111 | log.Infof("output is closed") 112 | } 113 | 114 | func (o *OutputPlugin) appendMsgTxnBuffer(msg *core.Msg) { 115 | key := metas.GenerateMapRouterVersionKey(msg.Database, msg.Table, msg.DmlMsg.TableVersion) 116 | o.msgTxnBuffer.tableMsgMap[key] = append(o.msgTxnBuffer.tableMsgMap[key], msg) 117 | o.msgTxnBuffer.size += 1 118 | } 119 | 120 | func (o *OutputPlugin) flushMsgTxnBuffer(pos core.Position) { 121 | defer func() { 122 | // flush position 123 | err := pos.Update(o.lastPosition) 124 | if err != nil { 125 | log.Fatalf(err.Error()) 126 | } 127 | }() 128 | 129 | if o.msgTxnBuffer.size == 0 { 130 | return 131 | } 132 | // table level send 133 | for k, msgs := range o.msgTxnBuffer.tableMsgMap { 134 | // columnsMapper := o.metas.Routers.Maps[k].ColumnsMapper 135 | schemaName, tableName, version := metas.SplitMapRouterVersionKey(k) 136 | dmlTopic := o.metas.Routers.Maps[metas.GenerateMapRouterKey(schemaName, tableName)].DmlTopic 137 | table, err := o.metas.Input.GetVersion(schemaName, tableName, version) 138 | if err != nil { 139 | log.Fatalf("get input table meta failed, err: %v", err.Error()) 140 | } 141 | err = o.execute(msgs, table, dmlTopic) 142 | if err != nil { 143 | log.Fatalf("output %s send err %v", PluginName, err) 144 | } 145 | } 146 | o.clearMsgTxnBuffer() 147 | } 148 | 149 | func (o *OutputPlugin) clearMsgTxnBuffer() { 150 | o.msgTxnBuffer.size = 0 151 | o.msgTxnBuffer.tableMsgMap = make(map[string][]*core.Msg) 152 | } 153 | 154 | func (o *OutputPlugin) execute(msgs []*core.Msg, table *metas.Table, dmlTopic string) error { 155 | for _, msg := range msgs { 156 | formatMsg := o.formatInterface.formatMsg(msg, table) 157 | bFormatMsg, err := json.Marshal(formatMsg) 158 | if err != nil { 159 | return err 160 | } 161 | pksData, err := GenPrimaryKeys(table.PrimaryKeyColumns, msg.DmlMsg.Data) 162 | if err != nil { 163 | return err 164 | } 165 | _, dataHash, err := DataHash(pksData) 166 | if err != nil { 167 | return err 168 | } 169 | kPartition := dataHash % uint64(o.PartitionNum) 170 | kKey := strconv.FormatUint(dataHash, 10) 171 | 172 | kMsg := gokafka.Message{ 173 | TopicPartition: gokafka.TopicPartition{Topic: &dmlTopic, Partition: int32(kPartition)}, 174 | Key: []byte(kKey), 175 | Value: bFormatMsg, 176 | Opaque: msg, 177 | } 178 | 179 | err = o.send(&kMsg) 180 | if err != nil { 181 | return err 182 | } 183 | log.Debugf("output %s msg: %v", PluginName, string(bFormatMsg)) 184 | // prom write event number counter 185 | metrics.OpsWriteProcessed.Add(1) 186 | } 187 | return nil 188 | } 189 | 190 | func (o *OutputPlugin) send(message *gokafka.Message) error { 191 | var err error 192 | for i := 0; i < RetryCount; i++ { 193 | err = o.client.Produce(message, nil) 194 | if err != nil { 195 | log.Warnf("kafka send data failed, err: %v, start retry...", err.Error()) 196 | if i+1 == RetryCount { 197 | break 198 | } 199 | time.Sleep(time.Duration(RetryInterval*(i+1)) * time.Second) 200 | continue 201 | } 202 | break 203 | } 204 | if err != nil { 205 | return err 206 | } 207 | return err 208 | } 209 | -------------------------------------------------------------------------------- /outputs/kafka/kafka_meta.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | gokafka "github.com/confluentinc/confluent-kafka-go/v2/kafka" 5 | "github.com/mitchellh/mapstructure" 6 | "github.com/sqlpub/qin-cdc/config" 7 | "github.com/sqlpub/qin-cdc/metas" 8 | "sync" 9 | ) 10 | 11 | type MetaPlugin struct { 12 | *config.KafkaConfig 13 | topics map[string]*Topic 14 | producer *gokafka.Producer 15 | mu sync.Mutex 16 | } 17 | 18 | type Topic struct { 19 | Name string 20 | Partition int 21 | } 22 | 23 | func (m *MetaPlugin) Configure(conf map[string]interface{}) error { 24 | m.KafkaConfig = &config.KafkaConfig{} 25 | var target = conf["target"] 26 | if err := mapstructure.Decode(target, m.KafkaConfig); err != nil { 27 | return err 28 | } 29 | return nil 30 | } 31 | 32 | func (m *MetaPlugin) LoadMeta(routers []*metas.Router) (err error) { 33 | m.topics = make(map[string]*Topic) 34 | m.producer, err = getProducer(m.KafkaConfig) 35 | if err != nil { 36 | return err 37 | } 38 | for _, router := range routers { 39 | dmlTopic := router.DmlTopic 40 | if ok := m.topics[dmlTopic]; ok != nil { 41 | m.topics[dmlTopic].Name = dmlTopic 42 | metadata, err := m.producer.GetMetadata(&dmlTopic, false, 3000) 43 | if err != nil { 44 | return err 45 | } 46 | m.topics[dmlTopic].Partition = len(metadata.Topics[dmlTopic].Partitions) 47 | } 48 | } 49 | return nil 50 | } 51 | 52 | func (m *MetaPlugin) GetMeta(router *metas.Router) (topic interface{}, err error) { 53 | return m.Get(router.DmlTopic) 54 | } 55 | 56 | func (m *MetaPlugin) Get(topicName string) (topic *Topic, err error) { 57 | return m.topics[topicName], err 58 | } 59 | 60 | func (m *MetaPlugin) GetAll() map[string]*Topic { 61 | m.mu.Lock() 62 | defer m.mu.Unlock() 63 | return m.topics 64 | } 65 | 66 | func (m *MetaPlugin) Add(newTopic *Topic) error { 67 | m.mu.Lock() 68 | defer m.mu.Unlock() 69 | m.topics[newTopic.Name] = newTopic 70 | return nil 71 | } 72 | 73 | func (m *MetaPlugin) Update(newTopic *Topic) error { 74 | m.mu.Lock() 75 | defer m.mu.Unlock() 76 | m.topics[newTopic.Name] = newTopic 77 | return nil 78 | } 79 | 80 | func (m *MetaPlugin) Delete(topicName string) error { 81 | m.mu.Lock() 82 | defer m.mu.Unlock() 83 | delete(m.topics, topicName) 84 | return nil 85 | } 86 | 87 | func (m *MetaPlugin) Save() error { 88 | return nil 89 | } 90 | 91 | func (m *MetaPlugin) Close() { 92 | closeProducer(m.producer) 93 | } 94 | -------------------------------------------------------------------------------- /outputs/kafka/kafka_utils.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "fmt" 5 | gokafka "github.com/confluentinc/confluent-kafka-go/v2/kafka" 6 | "github.com/juju/errors" 7 | "github.com/mitchellh/hashstructure/v2" 8 | "github.com/siddontang/go-log/log" 9 | "github.com/sqlpub/qin-cdc/config" 10 | "github.com/sqlpub/qin-cdc/core" 11 | "github.com/sqlpub/qin-cdc/metas" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | type formatType string 17 | 18 | var inputSequence uint64 19 | 20 | const ( 21 | PluginName = "kafka" 22 | DefaultBatchSize int = 10240 23 | DefaultBatchIntervalMs int = 100 24 | RetryCount int = 3 25 | RetryInterval int = 5 26 | defaultJson formatType = "json" 27 | aliyunDtsCanal formatType = "aliyun_dts_canal" 28 | ) 29 | 30 | func getProducer(conf *config.KafkaConfig) (producer *gokafka.Producer, err error) { 31 | kafkaConf := &gokafka.ConfigMap{ 32 | "api.version.request": "true", 33 | "message.max.bytes": 1000000, 34 | "linger.ms": 10, 35 | "retries": 30, 36 | "retry.backoff.ms": 1000, 37 | "acks": "1"} 38 | err = kafkaConf.SetKey("bootstrap.servers", strings.Join(conf.Brokers, ",")) 39 | if err != nil { 40 | return nil, err 41 | } 42 | err = kafkaConf.SetKey("security.protocol", "plaintext") 43 | if err != nil { 44 | return nil, err 45 | } 46 | producer, err = gokafka.NewProducer(kafkaConf) 47 | return producer, err 48 | } 49 | 50 | func closeProducer(producer *gokafka.Producer) { 51 | if producer != nil { 52 | producer.Close() 53 | } 54 | } 55 | 56 | type formatInterface interface { 57 | formatMsg(event *core.Msg, table *metas.Table) interface{} 58 | } 59 | 60 | func (o *OutputPlugin) initFormatPlugin(outputFormat string) { 61 | // init kafka format handle func 62 | outputFormatType := formatType(fmt.Sprintf("%v", outputFormat)) 63 | switch outputFormatType { 64 | case defaultJson: 65 | o.formatInterface = &defaultJsonFormat{} 66 | case aliyunDtsCanal: 67 | o.formatInterface = &aliyunDtsCanalFormat{} 68 | default: 69 | log.Fatalf("Unknown format type: %v", outputFormatType) 70 | } 71 | } 72 | 73 | type defaultJsonFormat struct{} 74 | 75 | type kafkaDefaultMsg struct { 76 | Database string `json:"database"` 77 | Table string `json:"table"` 78 | Type core.ActionType `json:"type"` 79 | Ts uint32 `json:"ts"` 80 | Data map[string]interface{} `json:"data"` 81 | Old map[string]interface{} `json:"old"` 82 | } 83 | 84 | func (djf *defaultJsonFormat) formatMsg(event *core.Msg, table *metas.Table) interface{} { 85 | kMsg := &kafkaDefaultMsg{ 86 | Database: event.Database, 87 | Table: event.Table, 88 | Type: event.DmlMsg.Action, 89 | Ts: uint32(event.Timestamp.Unix()), 90 | Data: event.DmlMsg.Data, 91 | Old: event.DmlMsg.Old, 92 | } 93 | return kMsg 94 | } 95 | 96 | type aliyunDtsCanalFormat struct{} 97 | 98 | type kafkaMsgForAliyunDtsCanal struct { 99 | Database string `json:"database"` 100 | Table string `json:"table"` 101 | Type string `json:"type"` 102 | Es uint64 `json:"es"` // source write datetime 103 | Ts uint64 `json:"ts"` // target write datetime 104 | Data []map[string]interface{} `json:"data"` 105 | Old []map[string]interface{} `json:"old"` 106 | SqlType map[string]interface{} `json:"sqlType"` 107 | MysqlType map[string]interface{} `json:"mysqlType"` 108 | ServerId string `json:"serverId"` 109 | Sql string `json:"sql"` 110 | PkNames []string `json:"pkNames"` 111 | IsDdl bool `json:"isDdl"` 112 | Id uint64 `json:"id"` 113 | Gtid *string `json:"gtid"` 114 | } 115 | 116 | func (adc *aliyunDtsCanalFormat) formatMsg(event *core.Msg, table *metas.Table) interface{} { 117 | datas := make([]map[string]interface{}, 0) 118 | datas = append(datas, event.DmlMsg.Data) 119 | var olds []map[string]interface{} 120 | if event.DmlMsg.Old != nil { 121 | olds = make([]map[string]interface{}, 0) 122 | olds = append(olds, event.DmlMsg.Old) 123 | } 124 | sqlType := make(map[string]interface{}) 125 | mysqlType := make(map[string]interface{}) 126 | for _, column := range table.Columns { 127 | if event.DmlMsg.Data[column.Name] != nil { 128 | event.DmlMsg.Data[column.Name] = fmt.Sprintf("%v", event.DmlMsg.Data[column.Name]) // to string 129 | } 130 | if event.DmlMsg.Old[column.Name] != nil { 131 | event.DmlMsg.Old[column.Name] = fmt.Sprintf("%v", event.DmlMsg.Old[column.Name]) // to string 132 | } 133 | switch column.Type { 134 | case metas.TypeNumber: // tinyint, smallint, int, bigint, year 135 | if strings.HasPrefix(column.RawType, "smallint") { 136 | sqlType[column.Name] = 2 137 | mysqlType[column.Name] = "smallint" 138 | continue 139 | } else if strings.HasPrefix(column.RawType, "tinyint") { 140 | sqlType[column.Name] = 1 141 | mysqlType[column.Name] = "tinyint" 142 | continue 143 | } else if strings.HasPrefix(column.RawType, "mediumint") { 144 | sqlType[column.Name] = 9 145 | mysqlType[column.Name] = "mediumint" 146 | continue 147 | } else if strings.HasPrefix(column.RawType, "bigint") { 148 | sqlType[column.Name] = 8 149 | mysqlType[column.Name] = "bigint" 150 | continue 151 | } else if strings.HasPrefix(column.RawType, "year") { 152 | mysqlType[column.Name] = "year" 153 | continue 154 | } else { 155 | sqlType[column.Name] = 3 156 | mysqlType[column.Name] = "int" 157 | continue 158 | } 159 | case metas.TypeFloat: // float, double 160 | if strings.HasPrefix(column.RawType, "float") { 161 | sqlType[column.Name] = 4 162 | mysqlType[column.Name] = "float" 163 | continue 164 | } else if strings.HasPrefix(column.RawType, "double") { 165 | sqlType[column.Name] = 5 166 | mysqlType[column.Name] = "double" 167 | continue 168 | } 169 | case metas.TypeEnum: // enum 170 | sqlType[column.Name] = 247 171 | mysqlType[column.Name] = "enum" 172 | continue 173 | case metas.TypeSet: // set 174 | sqlType[column.Name] = 248 175 | mysqlType[column.Name] = "set" 176 | continue 177 | case metas.TypeString: // other 178 | if strings.HasSuffix(column.RawType, "text") { 179 | sqlType[column.Name] = 15 180 | mysqlType[column.Name] = "text" 181 | continue 182 | } else if strings.HasPrefix(column.RawType, "char") { 183 | sqlType[column.Name] = 254 184 | mysqlType[column.Name] = "char" 185 | continue 186 | } else { 187 | sqlType[column.Name] = 253 188 | mysqlType[column.Name] = "varchar" 189 | continue 190 | } 191 | case metas.TypeDatetime: // datetime 192 | sqlType[column.Name] = 12 193 | mysqlType[column.Name] = "datetime" 194 | continue 195 | case metas.TypeTimestamp: // timestamp 196 | sqlType[column.Name] = 7 197 | mysqlType[column.Name] = "timestamp" 198 | continue 199 | case metas.TypeDate: // date 200 | sqlType[column.Name] = 10 201 | mysqlType[column.Name] = "date" 202 | continue 203 | case metas.TypeTime: // time 204 | sqlType[column.Name] = 11 205 | mysqlType[column.Name] = "time" 206 | continue 207 | case metas.TypeBit: // bit 208 | sqlType[column.Name] = 16 209 | mysqlType[column.Name] = "bit" 210 | continue 211 | case metas.TypeJson: // json 212 | sqlType[column.Name] = 245 213 | mysqlType[column.Name] = "json" 214 | continue 215 | case metas.TypeDecimal: // decimal 216 | sqlType[column.Name] = 246 217 | mysqlType[column.Name] = "decimal" 218 | continue 219 | case metas.TypeBinary: 220 | sqlType[column.Name] = 252 221 | if strings.HasPrefix(column.RawType, "binary") { 222 | mysqlType[column.Name] = "binary" 223 | } else { 224 | mysqlType[column.Name] = "blob" 225 | } 226 | continue 227 | default: 228 | sqlType[column.Name] = column.Type 229 | mysqlType[column.Name] = column.RawType 230 | } 231 | } 232 | 233 | pkNames := make([]string, 0) 234 | for _, primaryKeyColumn := range table.PrimaryKeyColumns { 235 | pkNames = append(pkNames, primaryKeyColumn.Name) 236 | } 237 | inputSequence++ 238 | kMsg := &kafkaMsgForAliyunDtsCanal{ 239 | Database: event.Database, 240 | Table: event.Table, 241 | Type: strings.ToUpper(string(event.DmlMsg.Action)), 242 | Es: uint64(event.Timestamp.UnixMilli()), 243 | Ts: uint64(time.Now().UnixMilli()), 244 | Data: datas, 245 | Old: olds, 246 | SqlType: sqlType, 247 | MysqlType: mysqlType, 248 | ServerId: "", 249 | Sql: "", 250 | PkNames: pkNames, 251 | IsDdl: false, 252 | Id: inputSequence, 253 | Gtid: nil, 254 | } 255 | return kMsg 256 | } 257 | 258 | func DataHash(key interface{}) (string, uint64, error) { 259 | hash, err := hashstructure.Hash(key, hashstructure.FormatV2, nil) 260 | if err != nil { 261 | return "", 0, err 262 | } 263 | return fmt.Sprint(key), hash, nil 264 | } 265 | 266 | func GenPrimaryKeys(pkColumns []metas.Column, rowData map[string]interface{}) (map[string]interface{}, error) { 267 | pks := make(map[string]interface{}) 268 | for i := 0; i < len(pkColumns); i++ { 269 | pkName := pkColumns[i].Name 270 | pks[pkName] = rowData[pkName] 271 | if pks[pkName] == nil { 272 | return nil, errors.Errorf("primary key nil, pkName: %v, data: %v", pkName, rowData) 273 | } 274 | } 275 | return pks, nil 276 | } 277 | -------------------------------------------------------------------------------- /outputs/mysql/mysql.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "github.com/juju/errors" 6 | "github.com/mitchellh/mapstructure" 7 | "github.com/siddontang/go-log/log" 8 | "github.com/sqlpub/qin-cdc/config" 9 | "github.com/sqlpub/qin-cdc/core" 10 | "github.com/sqlpub/qin-cdc/metas" 11 | "github.com/sqlpub/qin-cdc/metrics" 12 | "time" 13 | ) 14 | 15 | type OutputPlugin struct { 16 | *config.MysqlConfig 17 | Done chan bool 18 | metas *core.Metas 19 | msgTxnBuffer struct { 20 | size int 21 | tableMsgMap map[string][]*core.Msg 22 | } 23 | client *sql.DB 24 | lastPosition string 25 | } 26 | 27 | func (o *OutputPlugin) Configure(conf map[string]interface{}) error { 28 | o.MysqlConfig = &config.MysqlConfig{} 29 | var targetConf = conf["target"] 30 | if err := mapstructure.Decode(targetConf, o.MysqlConfig); err != nil { 31 | return err 32 | } 33 | return nil 34 | } 35 | 36 | func (o *OutputPlugin) NewOutput(metas *core.Metas) { 37 | o.Done = make(chan bool) 38 | o.metas = metas 39 | // options handle 40 | if o.MysqlConfig.Options.BatchSize == 0 { 41 | o.MysqlConfig.Options.BatchSize = DefaultBatchSize 42 | } 43 | if o.MysqlConfig.Options.BatchIntervalMs == 0 { 44 | o.MysqlConfig.Options.BatchIntervalMs = DefaultBatchIntervalMs 45 | } 46 | o.msgTxnBuffer.size = 0 47 | o.msgTxnBuffer.tableMsgMap = make(map[string][]*core.Msg) 48 | 49 | var err error 50 | o.client, err = getConn(o.MysqlConfig) 51 | if err != nil { 52 | log.Fatal("output config client failed. err: ", err.Error()) 53 | } 54 | } 55 | 56 | func (o *OutputPlugin) Start(out chan *core.Msg, pos core.Position) { 57 | // first pos 58 | o.lastPosition = pos.Get() 59 | go func() { 60 | ticker := time.NewTicker(time.Millisecond * time.Duration(o.Options.BatchIntervalMs)) 61 | defer ticker.Stop() 62 | for { 63 | select { 64 | case data := <-out: 65 | switch data.Type { 66 | case core.MsgCtl: 67 | o.lastPosition = data.InputContext.Pos 68 | case core.MsgDML: 69 | o.appendMsgTxnBuffer(data) 70 | if o.msgTxnBuffer.size >= o.MysqlConfig.Options.BatchSize { 71 | o.flushMsgTxnBuffer(pos) 72 | } 73 | } 74 | case <-ticker.C: 75 | o.flushMsgTxnBuffer(pos) 76 | case <-o.Done: 77 | o.flushMsgTxnBuffer(pos) 78 | return 79 | } 80 | 81 | } 82 | }() 83 | } 84 | 85 | func (o *OutputPlugin) Close() { 86 | log.Infof("output is closing...") 87 | close(o.Done) 88 | <-o.Done 89 | closeConn(o.client) 90 | log.Infof("output is closed") 91 | } 92 | 93 | func (o *OutputPlugin) appendMsgTxnBuffer(msg *core.Msg) { 94 | key := metas.GenerateMapRouterKey(msg.Database, msg.Table) 95 | o.msgTxnBuffer.tableMsgMap[key] = append(o.msgTxnBuffer.tableMsgMap[key], msg) 96 | o.msgTxnBuffer.size += 1 97 | } 98 | 99 | func (o *OutputPlugin) flushMsgTxnBuffer(pos core.Position) { 100 | defer func() { 101 | // flush position 102 | err := pos.Update(o.lastPosition) 103 | if err != nil { 104 | log.Fatalf(err.Error()) 105 | } 106 | }() 107 | 108 | if o.msgTxnBuffer.size == 0 { 109 | return 110 | } 111 | // table level export 112 | for k, msgs := range o.msgTxnBuffer.tableMsgMap { 113 | columnsMapper := o.metas.Routers.Maps[k].ColumnsMapper 114 | targetSchema := o.metas.Routers.Maps[k].TargetSchema 115 | targetTable := o.metas.Routers.Maps[k].TargetTable 116 | err := o.execute(msgs, columnsMapper, targetSchema, targetTable) 117 | if err != nil { 118 | log.Fatalf("do %s bulk err %v", PluginName, err) 119 | } 120 | } 121 | o.clearMsgTxnBuffer() 122 | } 123 | 124 | func (o *OutputPlugin) clearMsgTxnBuffer() { 125 | o.msgTxnBuffer.size = 0 126 | o.msgTxnBuffer.tableMsgMap = make(map[string][]*core.Msg) 127 | } 128 | 129 | func (o *OutputPlugin) execute(msgs []*core.Msg, columnsMapper metas.ColumnsMapper, targetSchema string, targetTable string) error { 130 | if len(columnsMapper.PrimaryKeys) == 0 { 131 | return errors.Errorf("only support data has primary key") 132 | } 133 | 134 | splitMsgsList := o.splitMsgs(msgs) 135 | for _, splitMsgs := range splitMsgsList { 136 | if splitMsgs[0].DmlMsg.Action != core.DeleteAction { 137 | // insert and update can bulk exec 138 | bulkSQL, args, err := o.generateBulkInsertOnDuplicateKeyUpdateSQL(splitMsgs, columnsMapper, targetSchema, targetTable) 139 | err = o.executeSQL(bulkSQL, args) 140 | if err != nil { 141 | return err 142 | } 143 | log.Debugf("output %s sql: %v; args: %v", PluginName, bulkSQL, args) 144 | } else { 145 | if len(columnsMapper.PrimaryKeys) > 1 { // multi-pk single sql exec (delete from table where pk1 = ? and pk2 = ? ...) 146 | for _, msg := range splitMsgs { 147 | singleSQL, args, err := o.generateSingleDeleteSQL(msg, columnsMapper, targetSchema, targetTable) 148 | err = o.executeSQL(singleSQL, args) 149 | if err != nil { 150 | return err 151 | } 152 | log.Debugf("output %s sql: %v; args: %v", PluginName, singleSQL, args) 153 | } 154 | 155 | } else { // one-pk bulk sql exec (delete from table where pk in (?,? ..)) 156 | bulkSQL, args, err := o.generateBulkDeleteSQL(splitMsgs, columnsMapper, targetSchema, targetTable) 157 | err = o.executeSQL(bulkSQL, args) 158 | if err != nil { 159 | return err 160 | } 161 | log.Debugf("output %s sql: %v; args: %v", PluginName, bulkSQL, args) 162 | } 163 | } 164 | // log.Debugf("%s bulk sync %s.%s row data num: %d", PluginName, targetSchema, targetTable, len(msgs)) 165 | 166 | // prom write event number counter 167 | metrics.OpsWriteProcessed.Add(float64(len(splitMsgs))) 168 | } 169 | return nil 170 | } 171 | 172 | func (o *OutputPlugin) splitMsgs(msgs []*core.Msg) [][]*core.Msg { 173 | msgsList := make([][]*core.Msg, 0) 174 | tmpMsgs := make([]*core.Msg, 0) 175 | tmpDeleteMsgs := make([]*core.Msg, 0) 176 | var nextMsgAction core.ActionType 177 | lenMsgs := len(msgs) 178 | // split delete msg 179 | for i, msg := range msgs { 180 | if i < lenMsgs-1 { 181 | nextMsgAction = msgs[i+1].DmlMsg.Action 182 | } else { 183 | // last msg 184 | nextMsgAction = "" 185 | } 186 | 187 | if msg.DmlMsg.Action == core.DeleteAction { 188 | tmpDeleteMsgs = append(tmpDeleteMsgs, msg) 189 | if nextMsgAction != core.DeleteAction { 190 | msgsList = append(msgsList, tmpDeleteMsgs) 191 | tmpDeleteMsgs = make([]*core.Msg, 0) 192 | } 193 | } else { 194 | tmpMsgs = append(tmpMsgs, msg) 195 | if nextMsgAction != core.InsertAction && nextMsgAction != core.UpdateAction { 196 | msgsList = append(msgsList, tmpMsgs) 197 | tmpMsgs = make([]*core.Msg, 0) 198 | } 199 | } 200 | } 201 | return msgsList 202 | } 203 | 204 | func (o *OutputPlugin) executeSQL(sqlCmd string, args []interface{}) error { 205 | var err error 206 | var result sql.Result 207 | for i := 0; i < RetryCount; i++ { 208 | result, err = o.client.Exec(sqlCmd, args...) 209 | if err != nil { 210 | log.Warnf("exec data failed, err: %v, execute retry...", err.Error()) 211 | if i+1 == RetryCount { 212 | break 213 | } 214 | time.Sleep(time.Duration(RetryInterval*(i+1)) * time.Second) 215 | continue 216 | } 217 | break 218 | } 219 | if err != nil { 220 | return err 221 | } 222 | if result == nil { 223 | return errors.Errorf("execute bulksql retry failed, result is nil") 224 | } 225 | return err 226 | } 227 | -------------------------------------------------------------------------------- /outputs/mysql/mysql_meta.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/mitchellh/mapstructure" 7 | "github.com/sqlpub/qin-cdc/config" 8 | "github.com/sqlpub/qin-cdc/metas" 9 | "strings" 10 | "sync" 11 | ) 12 | 13 | type MetaPlugin struct { 14 | *config.MysqlConfig 15 | tables map[string]*metas.Table 16 | tablesVersion map[string]*metas.Table 17 | db *sql.DB 18 | mu sync.Mutex 19 | } 20 | 21 | func (m *MetaPlugin) Configure(conf map[string]interface{}) error { 22 | m.MysqlConfig = &config.MysqlConfig{} 23 | var target = conf["target"] 24 | if err := mapstructure.Decode(target, m.MysqlConfig); err != nil { 25 | return err 26 | } 27 | return nil 28 | } 29 | 30 | func (m *MetaPlugin) LoadMeta(routers []*metas.Router) (err error) { 31 | m.tables = make(map[string]*metas.Table) 32 | m.tablesVersion = make(map[string]*metas.Table) 33 | m.db, err = getConn(m.MysqlConfig) 34 | if err != nil { 35 | return err 36 | } 37 | for _, router := range routers { 38 | row := m.db.QueryRow(fmt.Sprintf("show create table `%s`.`%s`", router.TargetSchema, router.TargetTable)) 39 | if row.Err() != nil { 40 | return err 41 | } 42 | var tableName string 43 | var createTableDdlStr string 44 | err = row.Scan(&tableName, &createTableDdlStr) 45 | if err != nil { 46 | return err 47 | } 48 | createTableDdlStr = strings.Replace(createTableDdlStr, "CREATE TABLE ", fmt.Sprintf("CREATE TABLE `%s`.", router.SourceSchema), 1) 49 | table := &metas.Table{} 50 | table, err = metas.NewTable(createTableDdlStr) 51 | if err != nil { 52 | return err 53 | } 54 | err = m.Add(table) 55 | if err != nil { 56 | return err 57 | } 58 | } 59 | return nil 60 | } 61 | 62 | func (m *MetaPlugin) GetMeta(router *metas.Router) (table interface{}, err error) { 63 | return m.Get(router.SourceSchema, router.SourceTable) 64 | } 65 | 66 | func (m *MetaPlugin) Get(schema string, tableName string) (table *metas.Table, err error) { 67 | key := metas.GenerateMapRouterKey(schema, tableName) 68 | m.mu.Lock() 69 | defer m.mu.Unlock() 70 | return m.tables[key], err 71 | } 72 | 73 | func (m *MetaPlugin) GetAll() map[string]*metas.Table { 74 | m.mu.Lock() 75 | defer m.mu.Unlock() 76 | return m.tables 77 | } 78 | 79 | func (m *MetaPlugin) GetVersion(schema string, tableName string, version uint) (table *metas.Table, err error) { 80 | key := metas.GenerateMapRouterVersionKey(schema, tableName, version) 81 | m.mu.Lock() 82 | defer m.mu.Unlock() 83 | return m.tablesVersion[key], err 84 | } 85 | 86 | func (m *MetaPlugin) GetVersions(schema string, tableName string) []*metas.Table { 87 | m.mu.Lock() 88 | defer m.mu.Unlock() 89 | tables := make([]*metas.Table, 0) 90 | for k, table := range m.tablesVersion { 91 | s, t, _ := metas.SplitMapRouterVersionKey(k) 92 | if schema == s && tableName == t { 93 | tables = append(tables, table) 94 | } 95 | } 96 | return tables 97 | } 98 | 99 | func (m *MetaPlugin) Add(newTable *metas.Table) error { 100 | m.mu.Lock() 101 | defer m.mu.Unlock() 102 | m.tables[metas.GenerateMapRouterKey(newTable.Schema, newTable.Name)] = newTable 103 | m.tablesVersion[metas.GenerateMapRouterVersionKey(newTable.Schema, newTable.Name, newTable.Version)] = newTable 104 | return nil 105 | } 106 | 107 | func (m *MetaPlugin) Update(newTable *metas.Table) error { 108 | m.mu.Lock() 109 | defer m.mu.Unlock() 110 | newTable.Version += 1 111 | m.tables[metas.GenerateMapRouterKey(newTable.Schema, newTable.Name)] = newTable 112 | m.tablesVersion[metas.GenerateMapRouterVersionKey(newTable.Schema, newTable.Name, newTable.Version)] = newTable 113 | return nil 114 | } 115 | 116 | func (m *MetaPlugin) Delete(schema string, name string) error { 117 | m.mu.Lock() 118 | defer m.mu.Unlock() 119 | delete(m.tables, metas.GenerateMapRouterKey(schema, name)) 120 | for _, table := range m.GetVersions(schema, name) { 121 | delete(m.tablesVersion, metas.GenerateMapRouterVersionKey(schema, name, table.Version)) 122 | } 123 | return nil 124 | } 125 | 126 | func (m *MetaPlugin) Save() error { 127 | return nil 128 | } 129 | 130 | func (m *MetaPlugin) Close() { 131 | closeConn(m.db) 132 | } 133 | -------------------------------------------------------------------------------- /outputs/mysql/mysql_utils.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | _ "github.com/go-sql-driver/mysql" 7 | "github.com/juju/errors" 8 | "github.com/siddontang/go-log/log" 9 | "github.com/sqlpub/qin-cdc/config" 10 | "github.com/sqlpub/qin-cdc/core" 11 | "github.com/sqlpub/qin-cdc/metas" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | const ( 17 | PluginName = "mysql" 18 | DefaultBatchSize int = 10240 19 | DefaultBatchIntervalMs int = 100 20 | RetryCount int = 3 21 | RetryInterval int = 5 22 | ) 23 | 24 | func getConn(conf *config.MysqlConfig) (db *sql.DB, err error) { 25 | dsn := fmt.Sprintf( 26 | "%s:%s@tcp(%s:%d)/information_schema?charset=utf8mb4&timeout=3s", 27 | conf.UserName, conf.Password, 28 | conf.Host, conf.Port) 29 | db, err = sql.Open("mysql", dsn) 30 | if err != nil { 31 | return db, err 32 | } 33 | db.SetConnMaxLifetime(time.Minute * 3) 34 | db.SetMaxOpenConns(2) 35 | db.SetMaxIdleConns(2) 36 | return db, err 37 | } 38 | 39 | func closeConn(db *sql.DB) { 40 | if db != nil { 41 | _ = db.Close() 42 | } 43 | } 44 | 45 | func (o *OutputPlugin) generateBulkInsertOnDuplicateKeyUpdateSQL(msgs []*core.Msg, columnsMapper metas.ColumnsMapper, targetSchema string, targetTable string) (string, []interface{}, error) { 46 | pks := make(map[string]interface{}, len(columnsMapper.PrimaryKeys)) 47 | for _, pk := range columnsMapper.PrimaryKeys { 48 | pks[pk] = nil 49 | } 50 | 51 | updateColumnsIdx := 0 52 | columnNamesAssignWithoutPks := make([]string, len(columnsMapper.MapMapper)-len(columnsMapper.PrimaryKeys)) 53 | allColumnNamesInSQL := make([]string, 0, len(columnsMapper.MapMapper)) 54 | allColumnPlaceHolder := make([]string, 0, len(columnsMapper.MapMapper)) 55 | for _, sourceColumn := range columnsMapper.MapMapperOrder { 56 | columnNameInSQL := fmt.Sprintf("`%s`", columnsMapper.MapMapper[sourceColumn]) 57 | allColumnNamesInSQL = append(allColumnNamesInSQL, columnNameInSQL) 58 | allColumnPlaceHolder = append(allColumnPlaceHolder, "?") 59 | _, ok := pks[sourceColumn] 60 | if !ok { 61 | columnNamesAssignWithoutPks[updateColumnsIdx] = fmt.Sprintf("%s = VALUES(%s)", columnNameInSQL, columnNameInSQL) 62 | updateColumnsIdx++ 63 | } 64 | } 65 | sqlInsert := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES ", 66 | targetSchema, 67 | targetTable, 68 | strings.Join(allColumnNamesInSQL, ",")) 69 | args := make([]interface{}, 0, len(columnsMapper.MapMapper)*len(msgs)) 70 | for i, msg := range msgs { 71 | switch msg.DmlMsg.Action { 72 | case core.InsertAction, core.UpdateAction: 73 | for _, sourceColumn := range columnsMapper.MapMapperOrder { 74 | columnData := msg.DmlMsg.Data[sourceColumn] 75 | args = append(args, columnData) 76 | } 77 | if i == 0 { 78 | sqlInsert += fmt.Sprintf("(%s)", strings.Join(allColumnPlaceHolder, ",")) 79 | } else { 80 | sqlInsert += fmt.Sprintf(",(%s)", strings.Join(allColumnPlaceHolder, ",")) 81 | } 82 | default: 83 | log.Fatalf("unhandled message type: %v", msg) 84 | } 85 | } 86 | sqlUpdate := fmt.Sprintf("ON DUPLICATE KEY UPDATE %s", strings.Join(columnNamesAssignWithoutPks, ",")) 87 | return fmt.Sprintf("%s %s", sqlInsert, sqlUpdate), args, nil 88 | } 89 | 90 | func (o *OutputPlugin) generateSingleDeleteSQL(msg *core.Msg, columnsMapper metas.ColumnsMapper, targetSchema string, targetTable string) (string, []interface{}, error) { 91 | pks := make(map[string]interface{}, len(columnsMapper.PrimaryKeys)) 92 | for _, pk := range columnsMapper.PrimaryKeys { 93 | pks[pk] = nil 94 | } 95 | 96 | var whereSql []string 97 | var args []interface{} 98 | for sourceColumn, targetColumn := range columnsMapper.MapMapper { 99 | pkData, ok := pks[sourceColumn] 100 | if !ok { 101 | continue 102 | } 103 | whereSql = append(whereSql, fmt.Sprintf("`%s` = ?", targetColumn)) 104 | args = append(args, pkData) 105 | } 106 | if len(whereSql) == 0 { 107 | return "", nil, errors.Errorf("where sql is empty, probably missing pk") 108 | } 109 | 110 | stmt := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE %s", targetSchema, targetTable, strings.Join(whereSql, " AND ")) 111 | return stmt, args, nil 112 | } 113 | 114 | func (o *OutputPlugin) generateBulkDeleteSQL(msgs []*core.Msg, columnsMapper metas.ColumnsMapper, targetSchema string, targetTable string) (string, []interface{}, error) { 115 | pkName := columnsMapper.PrimaryKeys[0] 116 | var whereSql []string 117 | var args []interface{} 118 | for _, msg := range msgs { 119 | pkData, ok := msg.DmlMsg.Data[pkName] 120 | if !ok { 121 | continue 122 | } 123 | whereSql = append(whereSql, "?") 124 | args = append(args, pkData) 125 | 126 | } 127 | targetPkName := columnsMapper.MapMapper[pkName] 128 | if len(whereSql) == 0 { 129 | return "", nil, errors.Errorf("where sql is empty, probably missing pk") 130 | } 131 | 132 | stmt := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE `%s` IN (%s)", targetSchema, targetTable, targetPkName, strings.Join(whereSql, ",")) 133 | return stmt, args, nil 134 | } 135 | -------------------------------------------------------------------------------- /outputs/starrocks/starrocks.go: -------------------------------------------------------------------------------- 1 | package starrocks 2 | 3 | import ( 4 | "fmt" 5 | "github.com/juju/errors" 6 | "github.com/mitchellh/mapstructure" 7 | "github.com/siddontang/go-log/log" 8 | "github.com/sqlpub/qin-cdc/config" 9 | "github.com/sqlpub/qin-cdc/core" 10 | "github.com/sqlpub/qin-cdc/metas" 11 | "github.com/sqlpub/qin-cdc/metrics" 12 | "io" 13 | "net/http" 14 | "strings" 15 | "time" 16 | ) 17 | 18 | type OutputPlugin struct { 19 | *config.StarrocksConfig 20 | Done chan bool 21 | metas *core.Metas 22 | msgTxnBuffer struct { 23 | size int 24 | tableMsgMap map[string][]*core.Msg 25 | } 26 | client *http.Client 27 | transport *http.Transport 28 | lastPosition string 29 | } 30 | 31 | func (o *OutputPlugin) Configure(conf map[string]interface{}) error { 32 | o.StarrocksConfig = &config.StarrocksConfig{} 33 | var targetConf = conf["target"] 34 | if err := mapstructure.Decode(targetConf, o.StarrocksConfig); err != nil { 35 | return err 36 | } 37 | return nil 38 | } 39 | 40 | func (o *OutputPlugin) NewOutput(metas *core.Metas) { 41 | o.Done = make(chan bool) 42 | o.metas = metas 43 | // options handle 44 | if o.StarrocksConfig.Options.BatchSize == 0 { 45 | o.StarrocksConfig.Options.BatchSize = DefaultBatchSize 46 | } 47 | if o.StarrocksConfig.Options.BatchIntervalMs == 0 { 48 | o.StarrocksConfig.Options.BatchIntervalMs = DefaultBatchIntervalMs 49 | } 50 | o.msgTxnBuffer.size = 0 51 | o.msgTxnBuffer.tableMsgMap = make(map[string][]*core.Msg) 52 | 53 | o.transport = &http.Transport{} 54 | o.client = &http.Client{ 55 | Transport: o.transport, 56 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 57 | req.Header.Add("Authorization", "Basic "+o.auth()) 58 | // log.Debugf("重定向请求到be: %v", req.URL) 59 | return nil // return nil nil回重定向。 60 | }, 61 | } 62 | } 63 | 64 | func (o *OutputPlugin) Start(out chan *core.Msg, pos core.Position) { 65 | // first pos 66 | o.lastPosition = pos.Get() 67 | go func() { 68 | ticker := time.NewTicker(time.Millisecond * time.Duration(o.Options.BatchIntervalMs)) 69 | defer ticker.Stop() 70 | for { 71 | select { 72 | case data := <-out: 73 | switch data.Type { 74 | case core.MsgCtl: 75 | o.lastPosition = data.InputContext.Pos 76 | case core.MsgDML: 77 | o.appendMsgTxnBuffer(data) 78 | if o.msgTxnBuffer.size >= o.StarrocksConfig.Options.BatchSize { 79 | o.flushMsgTxnBuffer(pos) 80 | } 81 | } 82 | case <-ticker.C: 83 | o.flushMsgTxnBuffer(pos) 84 | case <-o.Done: 85 | o.flushMsgTxnBuffer(pos) 86 | return 87 | } 88 | 89 | } 90 | }() 91 | } 92 | 93 | func (o *OutputPlugin) Close() { 94 | log.Infof("output is closing...") 95 | close(o.Done) 96 | <-o.Done 97 | log.Infof("output is closed") 98 | } 99 | 100 | func (o *OutputPlugin) appendMsgTxnBuffer(msg *core.Msg) { 101 | key := metas.GenerateMapRouterKey(msg.Database, msg.Table) 102 | o.msgTxnBuffer.tableMsgMap[key] = append(o.msgTxnBuffer.tableMsgMap[key], msg) 103 | o.msgTxnBuffer.size += 1 104 | } 105 | 106 | func (o *OutputPlugin) flushMsgTxnBuffer(pos core.Position) { 107 | defer func() { 108 | // flush position 109 | err := pos.Update(o.lastPosition) 110 | if err != nil { 111 | log.Fatalf(err.Error()) 112 | } 113 | }() 114 | 115 | if o.msgTxnBuffer.size == 0 { 116 | return 117 | } 118 | // table level export 119 | for k, msgs := range o.msgTxnBuffer.tableMsgMap { 120 | columnsMapper := o.metas.Routers.Maps[k].ColumnsMapper 121 | targetSchema := o.metas.Routers.Maps[k].TargetSchema 122 | targetTable := o.metas.Routers.Maps[k].TargetTable 123 | err := o.execute(msgs, columnsMapper, targetSchema, targetTable) 124 | if err != nil { 125 | log.Fatalf("do %s bulk err %v", PluginName, err) 126 | } 127 | } 128 | o.clearMsgTxnBuffer() 129 | } 130 | 131 | func (o *OutputPlugin) clearMsgTxnBuffer() { 132 | o.msgTxnBuffer.size = 0 133 | o.msgTxnBuffer.tableMsgMap = make(map[string][]*core.Msg) 134 | } 135 | 136 | func (o *OutputPlugin) execute(msgs []*core.Msg, columnsMapper metas.ColumnsMapper, targetSchema string, targetTable string) error { 137 | if len(msgs) == 0 { 138 | return nil 139 | } 140 | var jsonList []string 141 | 142 | jsonList = o.generateJson(msgs) 143 | for _, s := range jsonList { 144 | log.Debugf("%s load %s.%s row data: %v", PluginName, targetSchema, targetTable, s) 145 | } 146 | log.Debugf("%s bulk load %s.%s row data num: %d", PluginName, targetSchema, targetTable, len(jsonList)) 147 | var err error 148 | for i := 0; i < RetryCount; i++ { 149 | err = o.sendData(jsonList, columnsMapper, targetSchema, targetTable) 150 | if err != nil { 151 | log.Warnf("send data failed, err: %v, execute retry...", err.Error()) 152 | if i+1 == RetryCount { 153 | break 154 | } 155 | time.Sleep(time.Duration(RetryInterval*(i+1)) * time.Second) 156 | continue 157 | } 158 | break 159 | } 160 | return err 161 | } 162 | 163 | func (o *OutputPlugin) sendData(content []string, columnsMapper metas.ColumnsMapper, targetSchema string, targetTable string) error { 164 | loadUrl := fmt.Sprintf("http://%s:%d/api/%s/%s/_stream_load", 165 | o.Host, o.LoadPort, targetSchema, targetTable) 166 | newContent := `[` + strings.Join(content, ",") + `]` 167 | req, _ := http.NewRequest("PUT", loadUrl, strings.NewReader(newContent)) 168 | 169 | // req.Header.Add 170 | req.Header.Add("Authorization", "Basic "+o.auth()) 171 | req.Header.Add("Expect", "100-continue") 172 | req.Header.Add("strict_mode", "true") 173 | // req.Header.Add("label", "39c25a5c-7000-496e-a98e-348a264c81de") 174 | req.Header.Add("format", "json") 175 | req.Header.Add("strip_outer_array", "true") 176 | 177 | var columnArray []string 178 | for _, column := range columnsMapper.SourceColumns { 179 | columnArray = append(columnArray, column) 180 | } 181 | columnArray = append(columnArray, DeleteColumn) 182 | columns := fmt.Sprintf("%s, __op = %s", strings.Join(columnArray, ","), DeleteColumn) 183 | req.Header.Add("columns", columns) 184 | 185 | response, err := o.client.Do(req) 186 | if err != nil { 187 | return err 188 | } 189 | defer func(Body io.ReadCloser) { 190 | _ = Body.Close() 191 | }(response.Body) 192 | returnMap, err := o.parseResponse(response) 193 | if err != nil { 194 | return err 195 | } 196 | if returnMap["Status"] != "Success" { 197 | message := returnMap["Message"] 198 | errorUrl := returnMap["ErrorURL"] 199 | errorMsg := message.(string) + 200 | fmt.Sprintf(", targetTable: %s.%s", targetSchema, targetTable) + 201 | fmt.Sprintf(", visit ErrorURL to view error details, ErrorURL: %s", errorUrl) 202 | return errors.New(errorMsg) 203 | } 204 | // prom write event number counter 205 | numberLoadedRows := returnMap["NumberLoadedRows"] 206 | metrics.OpsWriteProcessed.Add(numberLoadedRows.(float64)) 207 | return nil 208 | } 209 | -------------------------------------------------------------------------------- /outputs/starrocks/starrocks_meta.go: -------------------------------------------------------------------------------- 1 | package starrocks 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "github.com/juju/errors" 7 | "github.com/mitchellh/mapstructure" 8 | "github.com/sqlpub/qin-cdc/config" 9 | "github.com/sqlpub/qin-cdc/metas" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | type MetaPlugin struct { 15 | *config.StarrocksConfig 16 | tables map[string]*metas.Table 17 | tablesVersion map[string]*metas.Table 18 | db *sql.DB 19 | mu sync.Mutex 20 | } 21 | 22 | func (m *MetaPlugin) Configure(conf map[string]interface{}) error { 23 | m.StarrocksConfig = &config.StarrocksConfig{} 24 | var target = conf["target"] 25 | if err := mapstructure.Decode(target, m.StarrocksConfig); err != nil { 26 | return err 27 | } 28 | return nil 29 | } 30 | 31 | func (m *MetaPlugin) LoadMeta(routers []*metas.Router) (err error) { 32 | m.tables = make(map[string]*metas.Table) 33 | m.tablesVersion = make(map[string]*metas.Table) 34 | dsn := fmt.Sprintf( 35 | "%s:%s@tcp(%s:%d)/information_schema?charset=utf8mb4&timeout=3s&interpolateParams=true", 36 | m.StarrocksConfig.UserName, m.StarrocksConfig.Password, 37 | m.StarrocksConfig.Host, m.StarrocksConfig.Port) 38 | m.db, err = sql.Open("mysql", dsn) 39 | if err != nil { 40 | return err 41 | } 42 | m.db.SetConnMaxLifetime(time.Minute * 3) 43 | m.db.SetMaxOpenConns(2) 44 | m.db.SetMaxIdleConns(2) 45 | for _, router := range routers { 46 | rows, err := m.db.Query("select "+ 47 | "column_name,column_default,is_nullable,data_type,column_type,column_key "+ 48 | "from information_schema.columns "+ 49 | "where table_schema = ? and table_name = ? "+ 50 | "order by ordinal_position", router.TargetSchema, router.TargetTable) 51 | if err != nil { 52 | return err 53 | } 54 | table := &metas.Table{ 55 | Schema: router.TargetSchema, 56 | Name: router.TargetTable, 57 | } 58 | for rows.Next() { 59 | var columnName, isNullable, dataType, columnType, columnKey string 60 | var columnDefault sql.NullString 61 | err = rows.Scan(&columnName, &columnDefault, &isNullable, &dataType, &columnType, &columnKey) 62 | if err != nil { 63 | return err 64 | } 65 | var column metas.Column 66 | column.Name = columnName 67 | column.RawType = columnType 68 | switch dataType { 69 | case "tinyint", "smallint", "mediumint", "int", "bigint": 70 | column.Type = metas.TypeNumber 71 | case "float", "double": 72 | column.Type = metas.TypeFloat 73 | case "enum": 74 | column.Type = metas.TypeEnum 75 | case "set": 76 | column.Type = metas.TypeSet 77 | case "datetime": 78 | column.Type = metas.TypeDatetime 79 | case "timestamp": 80 | column.Type = metas.TypeTimestamp 81 | case "date": 82 | column.Type = metas.TypeDate 83 | case "time": 84 | column.Type = metas.TypeTime 85 | case "bit": 86 | column.Type = metas.TypeBit 87 | case "json": 88 | column.Type = metas.TypeJson 89 | case "decimal": 90 | column.Type = metas.TypeDecimal 91 | default: 92 | column.Type = metas.TypeString 93 | } 94 | if columnKey == "PRI" { 95 | column.IsPrimaryKey = true 96 | } 97 | table.Columns = append(table.Columns, column) 98 | } 99 | if table.Columns == nil { 100 | return errors.Errorf("load meta %s.%s not found", router.TargetSchema, router.TargetTable) 101 | } 102 | err = m.Add(table) 103 | if err != nil { 104 | return err 105 | } 106 | } 107 | return nil 108 | } 109 | 110 | func (m *MetaPlugin) GetMeta(router *metas.Router) (table interface{}, err error) { 111 | return m.Get(router.SourceSchema, router.SourceTable) 112 | } 113 | 114 | func (m *MetaPlugin) Get(schema string, tableName string) (table *metas.Table, err error) { 115 | key := metas.GenerateMapRouterKey(schema, tableName) 116 | m.mu.Lock() 117 | defer m.mu.Unlock() 118 | return m.tables[key], err 119 | } 120 | 121 | func (m *MetaPlugin) GetAll() map[string]*metas.Table { 122 | m.mu.Lock() 123 | defer m.mu.Unlock() 124 | return m.tables 125 | } 126 | 127 | func (m *MetaPlugin) GetVersion(schema string, tableName string, version uint) (table *metas.Table, err error) { 128 | key := metas.GenerateMapRouterVersionKey(schema, tableName, version) 129 | m.mu.Lock() 130 | defer m.mu.Unlock() 131 | return m.tablesVersion[key], err 132 | } 133 | 134 | func (m *MetaPlugin) GetVersions(schema string, tableName string) []*metas.Table { 135 | m.mu.Lock() 136 | defer m.mu.Unlock() 137 | tables := make([]*metas.Table, 0) 138 | for k, table := range m.tablesVersion { 139 | s, t, _ := metas.SplitMapRouterVersionKey(k) 140 | if schema == s && tableName == t { 141 | tables = append(tables, table) 142 | } 143 | } 144 | return tables 145 | } 146 | 147 | func (m *MetaPlugin) Add(newTable *metas.Table) error { 148 | m.mu.Lock() 149 | defer m.mu.Unlock() 150 | m.tables[metas.GenerateMapRouterKey(newTable.Schema, newTable.Name)] = newTable 151 | m.tablesVersion[metas.GenerateMapRouterVersionKey(newTable.Schema, newTable.Name, newTable.Version)] = newTable 152 | return nil 153 | } 154 | 155 | func (m *MetaPlugin) Update(newTable *metas.Table) error { 156 | m.mu.Lock() 157 | defer m.mu.Unlock() 158 | newTable.Version += 1 159 | m.tables[metas.GenerateMapRouterKey(newTable.Schema, newTable.Name)] = newTable 160 | m.tablesVersion[metas.GenerateMapRouterVersionKey(newTable.Schema, newTable.Name, newTable.Version)] = newTable 161 | return nil 162 | } 163 | 164 | func (m *MetaPlugin) Delete(schema string, name string) error { 165 | m.mu.Lock() 166 | defer m.mu.Unlock() 167 | delete(m.tables, metas.GenerateMapRouterKey(schema, name)) 168 | for _, table := range m.GetVersions(schema, name) { 169 | delete(m.tablesVersion, metas.GenerateMapRouterVersionKey(schema, name, table.Version)) 170 | } 171 | return nil 172 | } 173 | 174 | func (m *MetaPlugin) Save() error { 175 | return nil 176 | } 177 | 178 | func (m *MetaPlugin) Close() { 179 | if m.db != nil { 180 | _ = m.db.Close() 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /outputs/starrocks/starrocks_utils.go: -------------------------------------------------------------------------------- 1 | package starrocks 2 | 3 | import ( 4 | "encoding/base64" 5 | "github.com/goccy/go-json" 6 | "github.com/siddontang/go-log/log" 7 | "github.com/sqlpub/qin-cdc/core" 8 | "io" 9 | "net/http" 10 | ) 11 | 12 | const ( 13 | PluginName = "starrocks" 14 | DefaultBatchSize int = 10240 15 | DefaultBatchIntervalMs int = 3000 16 | DeleteColumn string = "_delete_sign_" 17 | RetryCount int = 3 18 | RetryInterval int = 5 19 | ) 20 | 21 | func (o *OutputPlugin) auth() string { 22 | s := o.UserName + ":" + o.Password 23 | b := []byte(s) 24 | 25 | sEnc := base64.StdEncoding.EncodeToString(b) 26 | return sEnc 27 | } 28 | 29 | func (o *OutputPlugin) parseResponse(response *http.Response) (map[string]interface{}, error) { 30 | var result map[string]interface{} 31 | body, err := io.ReadAll(response.Body) 32 | if err == nil { 33 | err = json.Unmarshal(body, &result) 34 | } 35 | 36 | return result, err 37 | } 38 | 39 | func (o *OutputPlugin) generateJson(msgs []*core.Msg) []string { 40 | var jsonList []string 41 | 42 | for _, event := range msgs { 43 | switch event.DmlMsg.Action { 44 | case core.InsertAction: 45 | // 增加虚拟列，标识操作类型 (stream load opType：UPSERT 0，DELETE：1) 46 | event.DmlMsg.Data[DeleteColumn] = 0 47 | b, _ := json.Marshal(event.DmlMsg.Data) 48 | jsonList = append(jsonList, string(b)) 49 | case core.UpdateAction: 50 | // 增加虚拟列，标识操作类型 (stream load opType：UPSERT 0，DELETE：1) 51 | event.DmlMsg.Data[DeleteColumn] = 0 52 | b, _ := json.Marshal(event.DmlMsg.Data) 53 | jsonList = append(jsonList, string(b)) 54 | case core.DeleteAction: // starrocks2.4版本只支持primary key模型load delete 55 | // 增加虚拟列，标识操作类型 (stream load opType：UPSERT 0，DELETE：1) 56 | event.DmlMsg.Data[DeleteColumn] = 1 57 | b, _ := json.Marshal(event.DmlMsg.Data) 58 | jsonList = append(jsonList, string(b)) 59 | case core.ReplaceAction: // for mongo 60 | // 增加虚拟列，标识操作类型 (stream load opType：UPSERT 0，DELETE：1) 61 | event.DmlMsg.Data[DeleteColumn] = 0 62 | b, _ := json.Marshal(event.DmlMsg.Data) 63 | jsonList = append(jsonList, string(b)) 64 | default: 65 | log.Fatalf("unhandled message type: %v", event) 66 | } 67 | } 68 | return jsonList 69 | } 70 | -------------------------------------------------------------------------------- /registry/registry.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "fmt" 5 | "github.com/juju/errors" 6 | "github.com/siddontang/go-log/log" 7 | "sync" 8 | ) 9 | 10 | type PluginType string 11 | 12 | const ( 13 | InputPlugin PluginType = "input" 14 | OutputPlugin PluginType = "output" 15 | MetaPlugin PluginType = "meta" 16 | PositionPlugin PluginType = "position" 17 | ) 18 | 19 | type Plugin interface { 20 | Configure(data map[string]interface{}) error 21 | } 22 | 23 | var registry map[PluginType]map[string]Plugin 24 | var mutex sync.Mutex 25 | 26 | func init() { 27 | registry = make(map[PluginType]map[string]Plugin) 28 | } 29 | 30 | func RegisterPlugin(pluginType PluginType, name string, v Plugin) { 31 | mutex.Lock() 32 | defer mutex.Unlock() 33 | 34 | log.Debugf("[RegisterPlugin] type: %v, name: %v", pluginType, name) 35 | 36 | _, ok := registry[pluginType] 37 | if !ok { 38 | registry[pluginType] = make(map[string]Plugin) 39 | } 40 | 41 | _, ok = registry[pluginType][name] 42 | if ok { 43 | panic(fmt.Sprintf("plugin already exists, type: %v, name: %v", pluginType, name)) 44 | } 45 | registry[pluginType][name] = v 46 | } 47 | 48 | func GetPlugin(pluginType PluginType, name string) (Plugin, error) { 49 | mutex.Lock() 50 | defer mutex.Unlock() 51 | 52 | if registry == nil { 53 | return nil, errors.Errorf("empty registry") 54 | } 55 | 56 | plugins, ok := registry[pluginType] 57 | if !ok { 58 | return nil, errors.Errorf("empty plugin type: %v, name: %v", pluginType, name) 59 | } 60 | p, ok := plugins[name] 61 | if !ok { 62 | return nil, errors.Errorf("empty plugin, type: %v, name: %v", pluginType, name) 63 | } 64 | log.Infof("load %v plugin: %v", pluginType, name) 65 | return p, nil 66 | } 67 | -------------------------------------------------------------------------------- /transforms/trans_delete_column.go: -------------------------------------------------------------------------------- 1 | package transforms 2 | 3 | import ( 4 | "fmt" 5 | "github.com/juju/errors" 6 | "github.com/sqlpub/qin-cdc/core" 7 | "github.com/sqlpub/qin-cdc/utils" 8 | ) 9 | 10 | const DeleteColumnTransName = "delete-column" 11 | 12 | type DeleteColumnTrans struct { 13 | name string 14 | matchSchema string 15 | matchTable string 16 | columns []string 17 | } 18 | 19 | func (dct *DeleteColumnTrans) NewTransform(config map[string]interface{}) error { 20 | columns := config["columns"] 21 | c, ok := utils.CastToSlice(columns) 22 | if !ok { 23 | return errors.Trace(errors.New("'column' should be an array")) 24 | } 25 | 26 | columnsString, err := utils.CastSliceInterfaceToSliceString(c) 27 | if err != nil { 28 | return errors.Trace(errors.New("'column' should be an array of string")) 29 | } 30 | dct.name = DeleteColumnTransName 31 | dct.matchSchema = fmt.Sprintf("%v", config["match-schema"]) 32 | dct.matchTable = fmt.Sprintf("%v", config["match-table"]) 33 | dct.columns = columnsString 34 | return nil 35 | } 36 | 37 | func (dct *DeleteColumnTrans) Transform(msg *core.Msg) bool { 38 | if dct.matchSchema == msg.Database && dct.matchTable == msg.Table { 39 | for _, column := range dct.columns { 40 | value := FindColumn(msg.DmlMsg.Data, column) 41 | if value != nil { 42 | delete(msg.DmlMsg.Data, column) 43 | } 44 | } 45 | } 46 | return false 47 | } 48 | -------------------------------------------------------------------------------- /transforms/trans_rename_column.go: -------------------------------------------------------------------------------- 1 | package transforms 2 | 3 | import ( 4 | "fmt" 5 | "github.com/juju/errors" 6 | "github.com/sqlpub/qin-cdc/core" 7 | "github.com/sqlpub/qin-cdc/utils" 8 | ) 9 | 10 | const RenameColumnTransName = "rename-column" 11 | 12 | type RenameColumnTrans struct { 13 | name string 14 | matchSchema string 15 | matchTable string 16 | columns []string 17 | renameAs []string 18 | } 19 | 20 | func (rct *RenameColumnTrans) NewTransform(config map[string]interface{}) error { 21 | columns, ok := config["columns"] 22 | if !ok { 23 | return errors.Trace(errors.New("'columns' is not configured")) 24 | } 25 | renameAs, ok := config["rename-as"] 26 | if !ok { 27 | return errors.Trace(errors.New("'rename-as' is not configured")) 28 | } 29 | 30 | c, ok := utils.CastToSlice(columns) 31 | if !ok { 32 | return errors.Trace(errors.New("'columns' should be an array")) 33 | } 34 | 35 | columnsString, err := utils.CastSliceInterfaceToSliceString(c) 36 | if err != nil { 37 | return errors.Trace(errors.New("'columns' should be an array of string")) 38 | } 39 | 40 | ra, ok := utils.CastToSlice(renameAs) 41 | if !ok { 42 | return errors.Trace(errors.New("'rename-as' should be an array")) 43 | } 44 | 45 | renameAsString, err := utils.CastSliceInterfaceToSliceString(ra) 46 | if err != nil { 47 | return errors.Trace(errors.New("'cast-as' should be an array of string")) 48 | } 49 | 50 | if len(c) != len(ra) { 51 | return errors.Trace(errors.New("'columns' should have the same length of 'rename-as'")) 52 | } 53 | 54 | rct.name = RenameColumnTransName 55 | rct.matchSchema = fmt.Sprintf("%v", config["match-schema"]) 56 | rct.matchTable = fmt.Sprintf("%v", config["match-table"]) 57 | rct.columns = columnsString 58 | rct.renameAs = renameAsString 59 | return nil 60 | } 61 | 62 | func (rct *RenameColumnTrans) Transform(msg *core.Msg) bool { 63 | if rct.matchSchema == msg.Database && rct.matchTable == msg.Table { 64 | for i, column := range rct.columns { 65 | value := FindColumn(msg.DmlMsg.Data, column) 66 | if value != nil { 67 | renameAsColumn := rct.renameAs[i] 68 | msg.DmlMsg.Data[renameAsColumn] = msg.DmlMsg.Data[column] 69 | delete(msg.DmlMsg.Data, column) 70 | } 71 | } 72 | } 73 | return false 74 | } 75 | -------------------------------------------------------------------------------- /transforms/transforms.go: -------------------------------------------------------------------------------- 1 | package transforms 2 | 3 | import ( 4 | "github.com/siddontang/go-log/log" 5 | "github.com/sqlpub/qin-cdc/config" 6 | "github.com/sqlpub/qin-cdc/core" 7 | "github.com/sqlpub/qin-cdc/metas" 8 | "github.com/sqlpub/qin-cdc/metrics" 9 | ) 10 | 11 | type MatcherTransforms []core.Transform 12 | 13 | func NewMatcherTransforms(transConfigs []config.TransformConfig, routers *metas.Routers) (matcher MatcherTransforms) { 14 | for _, tc := range transConfigs { 15 | switch typ := tc.Type; typ { 16 | case RenameColumnTransName: 17 | rct := &RenameColumnTrans{} 18 | if err := rct.NewTransform(tc.Config); err != nil { 19 | log.Fatal(err) 20 | } 21 | // rename router mapper column name to new column name 22 | for _, router := range routers.Raws { 23 | if router.SourceSchema == rct.matchSchema && router.SourceTable == rct.matchTable { 24 | for i, column := range rct.columns { 25 | for i2, sourceColumn := range router.ColumnsMapper.SourceColumns { 26 | if sourceColumn == column { 27 | router.ColumnsMapper.SourceColumns[i2] = rct.renameAs[i] 28 | } 29 | } 30 | } 31 | } 32 | } 33 | log.Infof("load transform: %s", RenameColumnTransName) 34 | matcher = append(matcher, rct) 35 | case DeleteColumnTransName: 36 | dct := &DeleteColumnTrans{} 37 | if err := dct.NewTransform(tc.Config); err != nil { 38 | log.Fatal(err) 39 | } 40 | // delete router mapper column name 41 | for _, router := range routers.Raws { 42 | if router.SourceSchema == dct.matchSchema && router.SourceTable == dct.matchTable { 43 | for _, column := range dct.columns { 44 | sourceColumns := make([]string, len(router.ColumnsMapper.SourceColumns)) 45 | copy(sourceColumns, router.ColumnsMapper.SourceColumns) 46 | for i2, sourceColumn := range sourceColumns { 47 | if sourceColumn == column { 48 | router.ColumnsMapper.SourceColumns = append(router.ColumnsMapper.SourceColumns[:i2], router.ColumnsMapper.SourceColumns[i2+1:]...) 49 | } 50 | } 51 | } 52 | } 53 | } 54 | log.Infof("load transform: %s", DeleteColumnTransName) 55 | matcher = append(matcher, dct) 56 | default: 57 | log.Warnf("transform: %s unhandled will not take effect", typ) 58 | } 59 | } 60 | return matcher 61 | } 62 | 63 | func (m MatcherTransforms) IterateTransforms(msg *core.Msg) bool { 64 | for _, trans := range m { 65 | if trans.Transform(msg) { 66 | log.Debugf("transform msg %v", msg.DmlMsg.Data) 67 | return true 68 | } 69 | } 70 | return false 71 | } 72 | 73 | func (m MatcherTransforms) Start(in chan *core.Msg, out chan *core.Msg) { 74 | go func() { 75 | for data := range in { 76 | log.Debugf(data.ToString()) 77 | if !m.IterateTransforms(data) { 78 | out <- data 79 | handleMetrics(data) 80 | } 81 | } 82 | }() 83 | } 84 | 85 | func handleMetrics(data *core.Msg) { 86 | if data.Type == core.MsgDML { 87 | metrics.OpsReadProcessed.Inc() 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /transforms/utils.go: -------------------------------------------------------------------------------- 1 | package transforms 2 | 3 | func FindColumn(data map[string]interface{}, name string) interface{} { 4 | if value, ok := data[name]; ok { 5 | return value 6 | } 7 | return nil 8 | } 9 | -------------------------------------------------------------------------------- /utils/daemon.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/sevlyar/go-daemon" 5 | "github.com/siddontang/go-log/log" 6 | ) 7 | 8 | func Daemon(inputParam *Help) { 9 | if *inputParam.Daemon { 10 | cntxt := &daemon.Context{ 11 | PidFileName: GetExecPath() + "/qin-cdc.pid", 12 | PidFilePerm: 0644, 13 | LogFileName: *inputParam.LogFile, 14 | LogFilePerm: 0640, 15 | WorkDir: "./", 16 | Umask: 027, 17 | } 18 | d, err := cntxt.Reborn() 19 | if err != nil { 20 | log.Fatal("daemon mode run failed, err: ", err) 21 | } 22 | 23 | if d != nil { 24 | return 25 | } 26 | defer func(cntxt *daemon.Context) { 27 | err = cntxt.Release() 28 | if err != nil { 29 | log.Fatal("daemon release failed, err: ", err) 30 | } 31 | }(cntxt) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /utils/file_path.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/siddontang/go-log/log" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | func GetExecPath() string { 10 | ex, err := os.Executable() 11 | if err != nil { 12 | log.Fatal("get exec path error: ", err) 13 | } 14 | exPath := filepath.Dir(ex) 15 | return exPath 16 | } 17 | -------------------------------------------------------------------------------- /utils/help.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "flag" 5 | "github.com/go-demo/version" 6 | "github.com/siddontang/go-log/log" 7 | "os" 8 | ) 9 | 10 | type Help struct { 11 | printVersion bool 12 | ConfigFile *string 13 | LogLevel *string 14 | LogFile *string 15 | Daemon *bool 16 | HttpPort *uint 17 | } 18 | 19 | func InitHelp() (help *Help) { 20 | help = &Help{} 21 | help.ConfigFile = flag.String("config", "", "config file") 22 | help.LogLevel = flag.String("level", "info", "log level") 23 | help.LogFile = flag.String("log-file", "qin-cdc.log", "log file") 24 | help.Daemon = flag.Bool("daemon", false, "daemon run, must specify param 'log-file'") 25 | help.HttpPort = flag.Uint("http-port", 7716, "http monitor port, curl http://localhost:7716/metrics") 26 | flag.BoolVar(&help.printVersion, "version", false, "print program build version") 27 | flag.Parse() 28 | if help.printVersion { 29 | version.PrintVersion() 30 | os.Exit(0) 31 | } 32 | log.Infof("starting version: %s", version.Version) 33 | return help 34 | } 35 | -------------------------------------------------------------------------------- /utils/http.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "github.com/prometheus/client_golang/prometheus/promhttp" 6 | "github.com/siddontang/go-log/log" 7 | "github.com/sqlpub/qin-cdc/api" 8 | "github.com/sqlpub/qin-cdc/metrics" 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | func StartHttp(inputParam *Help) { 14 | // Start prometheus http monitor 15 | go func() { 16 | metrics.OpsStartTime.Set(float64(time.Now().Unix())) 17 | log.Infof("starting http on port: %d", *inputParam.HttpPort) 18 | http.Handle("/metrics", promhttp.Handler()) 19 | httpPortAddr := fmt.Sprintf(":%d", *inputParam.HttpPort) 20 | err := http.ListenAndServe(httpPortAddr, nil) 21 | if err != nil { 22 | log.Fatalf("starting http monitor error: %v", err) 23 | } 24 | }() 25 | } 26 | 27 | func InitHttpApi() { 28 | http.HandleFunc("/api/addRouter", api.AddRouter()) 29 | http.HandleFunc("/api/delRule", api.DelRouter()) 30 | http.HandleFunc("/api/getRule", api.GetRouter()) 31 | http.HandleFunc("/api/pause", api.PauseRouter()) 32 | http.HandleFunc("/api/resume", api.ResumeRouter()) 33 | } 34 | -------------------------------------------------------------------------------- /utils/input_param.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/siddontang/go-log/log" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | func InputParamHandle(inputParam *Help) { 10 | if *inputParam.ConfigFile == "" { 11 | log.Infof("-config param does not exist!") 12 | os.Exit(0) 13 | } else { 14 | abs, err := filepath.Abs(*inputParam.ConfigFile) 15 | if err != nil { 16 | log.Fatal("-config abs error: ", err.Error()) 17 | } 18 | *inputParam.ConfigFile = abs 19 | } 20 | if *inputParam.Daemon { 21 | if *inputParam.LogFile == "" { 22 | log.Infof("daemon mode, must specify -log-file param!") 23 | os.Exit(0) 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /utils/type_cast.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "github.com/juju/errors" 5 | "reflect" 6 | ) 7 | 8 | func CastToSlice(arg interface{}) (out []interface{}, ok bool) { 9 | slice, success := TakeArg(arg, reflect.Slice) 10 | if !success { 11 | ok = false 12 | return 13 | } 14 | c := slice.Len() 15 | out = make([]interface{}, c) 16 | for i := 0; i < c; i++ { 17 | out[i] = slice.Index(i).Interface() 18 | } 19 | return out, true 20 | } 21 | 22 | func TakeArg(arg interface{}, kind reflect.Kind) (val reflect.Value, ok bool) { 23 | val = reflect.ValueOf(arg) 24 | if val.Kind() == kind { 25 | ok = true 26 | } 27 | return 28 | } 29 | 30 | func CastSliceInterfaceToSliceString(a []interface{}) ([]string, error) { 31 | aStrings := make([]string, len(a)) 32 | for i, c := range a { 33 | name, ok := c.(string) 34 | if !ok { 35 | return nil, errors.Trace(errors.New("should be an array of string")) 36 | } 37 | aStrings[i] = name 38 | } 39 | return aStrings, nil 40 | } 41 | --------------------------------------------------------------------------------