├── .coveralls.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── bin ├── ccat_darwin └── ccat_linux ├── composer.json ├── phpunit.xml ├── src ├── Client.php ├── Cluster.php ├── Common │ ├── AbstractFile.php │ ├── File.php │ ├── FileFromString.php │ ├── Format.php │ ├── MergedFiles.php │ ├── Protocol.php │ ├── Sanitizer.php │ ├── ServerOptions.php │ └── TempTable.php ├── Exceptions │ ├── ClientException.php │ ├── ClusterException.php │ ├── QueryStatisticException.php │ ├── ResultException.php │ ├── ServerProviderException.php │ └── TransportException.php ├── Interfaces │ ├── FileInterface.php │ └── TransportInterface.php ├── Query.php ├── Query │ ├── QueryStatistic.php │ └── Result.php ├── Server.php ├── ServerProvider.php ├── Support │ └── CcatStream.php └── Transport │ └── HttpTransport.php ├── tests ├── CcatStreamTest.php ├── ClientTest.php ├── ClusterTest.php ├── FileFromStringTest.php ├── FileTest.php ├── HttpTransportTest.php ├── MergeFilesStreamTest.php ├── QueryStatisticTest.php ├── QueryTest.php ├── ResultTest.php ├── SanitizerTest.php ├── ServerOptionsTest.php ├── ServerProviderTest.php ├── ServerTest.php └── TempTableTest.php └── utils └── ccat ├── Makefile └── ccat.cpp /.coveralls.yml: -------------------------------------------------------------------------------- 1 | coverage_clover: tests/logs/clover.xml 2 | json_path: tests/logs/coveralls-upload.json 3 | service_name: travis-ci -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor 2 | /.idea 3 | /coverage 4 | /utils/ccat/ccat 5 | /utils/ccat/ccat_darwin 6 | /utils/ccat/ccat_linux 7 | composer.lock 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | 3 | php: 4 | - '7.1' 5 | - '7.2' 6 | - '7.3' 7 | - '7.4' 8 | - '8.0' 9 | 10 | sudo: required 11 | 12 | services: 13 | - docker 14 | 15 | compiler: 16 | - clang 17 | - g++ 18 | 19 | before_script: 20 | - composer self-update 21 | - composer install --prefer-source --dev 22 | - docker run -p 9000:9000 -p 8123:8123 -d --name some-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server 23 | - cd utils/ccat 24 | - make && make install 25 | - cd ../../ 26 | 27 | script: vendor/bin/phpunit --coverage-clover ./tests/logs/clover.xml 28 | 29 | after_script: 30 | - php vendor/bin/php-coveralls -v 31 | 32 | notifications: 33 | on_success: never 34 | on_failure: always 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | {one line to give the program's name and a brief idea of what it does.} 635 | Copyright (C) {year} {name of author} 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | {project} Copyright (C) {year} {fullname} 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Clickhouse Client 2 | [![Build Status](https://travis-ci.org/the-tinderbox/ClickhouseClient.svg?branch=master&200)](https://travis-ci.org/the-tinderbox/ClickhouseClient) [![Coverage Status](https://coveralls.io/repos/github/the-tinderbox/ClickhouseClient/badge.svg?branch=master&200)](https://coveralls.io/github/the-tinderbox/ClickhouseClient?branch=master) 3 | 4 | Package was written as client for [Clickhouse](https://clickhouse.yandex/). 5 | 6 | Client uses [Guzzle](https://github.com/guzzle/guzzle) for sending Http requests to Clickhouse servers. 7 | 8 | ## Requirements 9 | `php7.1` 10 | 11 | ## Install 12 | 13 | Composer 14 | 15 | ```bash 16 | composer require the-tinderbox/clickhouse-php-client 17 | ``` 18 | 19 | # Usage 20 | 21 | Client works with alone server and cluster. Also, client can make async select and insert (from local files) queries. 22 | 23 | ## Alone server 24 | 25 | ```php 26 | $server = new Tinderbox\Clickhouse\Server('127.0.0.1', '8123', 'default', 'user', 'pass'); 27 | $serverProvider = (new Tinderbox\Clickhouse\ServerProvider())->addServer($server); 28 | 29 | $client = new Tinderbox\Clickhouse\Client($serverProvider); 30 | ``` 31 | 32 | ## Cluster 33 | 34 | ```php 35 | $testCluster = new Tinderbox\Clickhouse\Cluster('cluster-name', [ 36 | 'server-1' => [ 37 | 'host' => '127.0.0.1', 38 | 'port' => '8123', 39 | 'database' => 'default', 40 | 'user' => 'user', 41 | 'password' => 'pass' 42 | ], 43 | 'server-2' => new Tinderbox\Clickhouse\Server('127.0.0.1', '8124', 'default', 'user', 'pass') 44 | ]); 45 | 46 | $anotherCluster = new Tinderbox\Clickhouse\Cluster('cluster-name', [ 47 | [ 48 | 'host' => '127.0.0.1', 49 | 'port' => '8125', 50 | 'database' => 'default', 51 | 'user' => 'user', 52 | 'password' => 'pass' 53 | ], 54 | new Tinderbox\Clickhouse\Server('127.0.0.1', '8126', 'default', 'user', 'pass') 55 | ]); 56 | 57 | $serverProvider = (new Tinderbox\Clickhouse\ServerProvider())->addCluster($testCluster)->addCluster($anotherCluster); 58 | 59 | $client = (new Tinderbox\Clickhouse\Client($serverProvider)); 60 | ``` 61 | 62 | Before execute any query on cluster, you should provide cluster name and client will run all queries on specified cluster. 63 | 64 | ``` 65 | $client->onCluster('test-cluster'); 66 | ``` 67 | 68 | By default client will use random server in given list of servers or in specified cluster. If you want to perform request on specified server you should use 69 | `using($hostname)` method on client and then run query. Client will remember hostname for next queries: 70 | 71 | ```php 72 | $client->using('server-2')->select('select * from table'); 73 | ``` 74 | 75 | ## Server tags 76 | 77 | ```php 78 | $firstServerOptionsWithTag = (new \Tinderbox\Clickhouse\Common\ServerOptions())->setTag('tag'); 79 | $secondServerOptionsWithAnotherTag = (new \Tinderbox\Clickhouse\Common\ServerOptions())->setTag('another-tag'); 80 | 81 | $server = new Tinderbox\Clickhouse\Server('127.0.0.1', '8123', 'default', 'user', 'pass', $firstServerOptionsWithTag); 82 | 83 | $cluster = new Tinderbox\Clickhouse\Cluster('cluster', [ 84 | new Tinderbox\Clickhouse\Server('127.0.0.2', '8123', 'default', 'user', 'pass', $secondServerOptionsWithAnotherTag) 85 | ]); 86 | 87 | $serverProvider = (new Tinderbox\Clickhouse\ServerProvider())->addServer($server)->addCluster($cluster); 88 | 89 | $client = (new Tinderbox\Clickhouse\Client($serverProvider)); 90 | ``` 91 | 92 | To use server with tag, you should call ```usingServerWithTag``` function before execute any query. 93 | 94 | ```php 95 | $client->usingServerWithTag('tag'); 96 | ``` 97 | 98 | ## Select queries 99 | 100 | Any SELECT query will return instance of `Result`. This class implements interfaces `\ArrayAccess`, `\Countable` и `\Iterator`, 101 | which means that it can be used as an array. 102 | 103 | Array with result rows can be obtained via `rows` property 104 | 105 | ```php 106 | $rows = $result->rows; 107 | $rows = $result->getRows(); 108 | ``` 109 | 110 | Also you can get some statistic of your query execution: 111 | 112 | 1. Number of read rows 113 | 2. Number of read bytes 114 | 3. Time of query execution 115 | 4. Rows before limit at least 116 | 117 | Statistic can be obtained via `statistic` property 118 | 119 | ```php 120 | $statistic = $result->statistic; 121 | $statistic = $result->getStatistic(); 122 | 123 | echo $statistic->rows; 124 | echo $statistic->getRows(); 125 | 126 | echo $statistic->bytes; 127 | echo $statistic->getBytes(); 128 | 129 | echo $statistic->time; 130 | echo $statistic->getTime(); 131 | 132 | echo $statistic->rowsBeforeLimitAtLeast; 133 | echo $statistic->getRowsBeforeLimitAtLeast(); 134 | ``` 135 | 136 | ### Sync 137 | 138 | ```php 139 | $result = $client->readOne('select number from system.numbers limit 100'); 140 | 141 | foreach ($result as $number) { 142 | echo $number['number'].PHP_EOL; 143 | } 144 | ``` 145 | 146 | **Using local files** 147 | 148 | You can use local files as temporary tables in Clickhouse. You should pass as third argument array of `TempTable` instances. 149 | instance. 150 | 151 | In this case will be sent one file to the server from which Clickhouse will extract data to temporary table. 152 | Structure of table will be: 153 | 154 | * number - UInt64 155 | 156 | If you pass such an array as a structure: 157 | 158 | ```php 159 | ['UInt64'] 160 | ``` 161 | 162 | Then each column from file wil be named as _1, _2, _3. 163 | 164 | ```php 165 | $result = $client->readOne('select number from system.numbers where number in _numbers limit 100', new TempTable('_numbers', 'numbers.csv', [ 166 | 'number' => 'UInt64' 167 | ])); 168 | 169 | foreach ($result as $number) { 170 | echo $number['number'].PHP_EOL; 171 | } 172 | ``` 173 | 174 | You can provide path to file or pass `FileInterface` instance as second argument. 175 | 176 | There is some other types of file streams which could be used to send to server: 177 | * File - simple file stored on disk; 178 | * FileFromString - stream created from string. For example: `new FileFromString('1'.PHP_EOL.'2'.PHP_EOL.'3'.PHP_EOL)` 179 | * MergedFiles - stream which includes many files and merges them all in one. 180 | You should pass to constructor file path, which contains list of files which 181 | should be megred in one stream. 182 | * TempTable - wrapper to any of `FileInterface` instance and contains structure. Usefull 183 | to make inserts using with `MergedFiles`. 184 | 185 | ### Async 186 | 187 | Unlike the `readOne` method, which returns` Result`, the `read` method returns an array of` Result` for each executed query. 188 | 189 | ```php 190 | 191 | list($clicks, $visits, $views) = $client->read([ 192 | ['query' => "select * from clicks where date = '2017-01-01'"], 193 | ['query' => "select * from visits where date = '2017-01-01'"], 194 | ['query' => "select * from views where date = '2017-01-01'"], 195 | ]); 196 | 197 | foreach ($clicks as $click) { 198 | echo $click['date'].PHP_EOL; 199 | } 200 | 201 | ``` 202 | **In `read` method, you can pass the parameter `$concurrency` which is responsible for the maximum simultaneous number of requests.** 203 | 204 | **Using local files** 205 | 206 | As with synchronous select request you can pass files to the server: 207 | 208 | ```php 209 | 210 | list($clicks, $visits, $views) = $client->read([ 211 | ['query' => "select * from clicks where date = '2017-01-01' and userId in _users", new TempTable('_users', 'users.csv', ['number' => 'UInt64'])], 212 | ['query' => "select * from visits where date = '2017-01-01'"], 213 | ['query' => "select * from views where date = '2017-01-01'"], 214 | ]); 215 | 216 | foreach ($clicks as $click) { 217 | echo $click['date'].PHP_EOL; 218 | } 219 | 220 | ``` 221 | 222 | With asynchronous requests you can pass multiple files as with synchronous request. 223 | 224 | ## Insert queries 225 | 226 | Insert queries always returns true or throws exceptions in case of error. 227 | 228 | Data can be written row by row or from local CSV or TSV files. 229 | 230 | ```php 231 | $client->writeOne("insert into table (date, column) values ('2017-01-01',1), ('2017-01-02',2)"); 232 | $client->write([ 233 | ['query' => "insert into table (date, column) values ('2017-01-01',1), ('2017-01-02',2)"], 234 | ['query' => "insert into table (date, column) values ('2017-01-01',1), ('2017-01-02',2)"], 235 | ['query' => "insert into table (date, column) values ('2017-01-01',1), ('2017-01-02',2)"] 236 | ]); 237 | 238 | $client->writeFiles('table', ['date', 'column'], [ 239 | new Tinderbox\Clickhouse\Common\File('/file-1.csv'), 240 | new Tinderbox\Clickhouse\Common\File('/file-2.csv') 241 | ]); 242 | 243 | $client->insertFiles('table', ['date', 'column'], [ 244 | new Tinderbox\Clickhouse\Common\File('/file-1.tsv'), 245 | new Tinderbox\Clickhouse\Common\File('/file-2.tsv') 246 | ], Tinderbox\Clickhouse\Common\Format::TSV); 247 | ``` 248 | 249 | In case of `writeFiles` queries executes asynchronously. If you have butch of files and you want to insert them in one insert query, you can 250 | use our `ccat` utility and `MergedFiles` instance instead of `File`. You should put list of files to insert into 251 | one file: 252 | 253 | ``` 254 | file-1.tsv 255 | file-2.tsv 256 | ``` 257 | 258 | ### Building ccat 259 | 260 | `ccat` sources placed into `utils/ccat` directory. Just run `make && make install` to build and install library into 261 | `bin` directory of package. There are already compiled binary of `ccat` in `bin` directory, but it 262 | may not work on some systems. 263 | 264 | **In `writeFiles` method, you can pass the parameter `$concurrency` which is responsible for the maximum simultaneous number of requests.** 265 | 266 | ## Other queries 267 | 268 | In addition to SELECT and INSERT queries, you can execute other queries :) There is `statement` method for this purposes. 269 | 270 | ```php 271 | $client->writeOne('DROP TABLE table'); 272 | ``` 273 | 274 | ## Testing 275 | 276 | ``` bash 277 | $ composer test 278 | ``` 279 | 280 | ## Roadmap 281 | 282 | * Add ability to save query result in local file 283 | 284 | ## Contributing 285 | Please send your own pull-requests and make suggestions on how to improve anything. 286 | We will be very grateful. 287 | 288 | Thx! 289 | -------------------------------------------------------------------------------- /bin/ccat_darwin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/the-tinderbox/ClickhouseClient/25c4e918bd30fb181add4c8db46e6d5aba510bf0/bin/ccat_darwin -------------------------------------------------------------------------------- /bin/ccat_linux: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/the-tinderbox/ClickhouseClient/25c4e918bd30fb181add4c8db46e6d5aba510bf0/bin/ccat_linux -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "the-tinderbox/clickhouse-php-client", 3 | "description": "Clickhouse client over HTTP", 4 | "authors": [ 5 | { 6 | "name": "FacedSID", 7 | "email": "ay@imagespark.ru" 8 | } 9 | ], 10 | "autoload": { 11 | "psr-4": { 12 | "Tinderbox\\Clickhouse\\": "src" 13 | } 14 | }, 15 | "autoload-dev": { 16 | "psr-4": { 17 | "Tinderbox\\Clickhouse\\": "tests" 18 | } 19 | }, 20 | "require": { 21 | "php": "^7.1|^8.0", 22 | "guzzlehttp/guzzle": "^6.0|^7.0" 23 | }, 24 | "bin": [ 25 | "bin/ccat_linux", 26 | "bin/ccat_darwin" 27 | ], 28 | "require-dev": { 29 | "mockery/mockery": "^0.9|^1.4", 30 | "phpunit/phpunit": "^7.0|^8.0|^9.0", 31 | "php-coveralls/php-coveralls": "^2.2", 32 | "phpunit/phpcov": "^5.0|^6.0|^7.0|^8.0" 33 | }, 34 | "scripts": { 35 | "test": "phpunit --coverage-text --colors=never" 36 | }, 37 | "config": { 38 | "sort-packages": true 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 12 | 13 | 14 | tests 15 | 16 | 17 | 18 | 19 | src/ 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /src/Client.php: -------------------------------------------------------------------------------- 1 | serverProvider = $serverProvider; 56 | $this->setTransport($transport); 57 | } 58 | 59 | /** 60 | * Creates default http transport. 61 | * 62 | * @return HttpTransport 63 | */ 64 | protected function createTransport() 65 | { 66 | return new HttpTransport(); 67 | } 68 | 69 | /** 70 | * Sets transport. 71 | * 72 | * @param \Tinderbox\Clickhouse\Interfaces\TransportInterface|null $transport 73 | */ 74 | protected function setTransport(TransportInterface $transport = null) 75 | { 76 | if (is_null($transport)) { 77 | $this->transport = $this->createTransport(); 78 | } else { 79 | $this->transport = $transport; 80 | } 81 | } 82 | 83 | /** 84 | * Returns transport. 85 | * 86 | * @return TransportInterface 87 | */ 88 | protected function getTransport(): TransportInterface 89 | { 90 | return $this->transport; 91 | } 92 | 93 | /** 94 | * Client will use servers from specified cluster. 95 | * 96 | * @param string|null $cluster 97 | * 98 | * @return $this 99 | */ 100 | public function onCluster(?string $cluster) 101 | { 102 | $this->clusterName = $cluster; 103 | $this->serverHostname = null; 104 | 105 | return $this; 106 | } 107 | 108 | /** 109 | * Returns current cluster name. 110 | * 111 | * @return null|string 112 | */ 113 | protected function getClusterName(): ?string 114 | { 115 | return $this->clusterName; 116 | } 117 | 118 | /** 119 | * Client will use specified server. 120 | * 121 | * @param string $serverHostname 122 | * 123 | * @return $this 124 | */ 125 | public function using(string $serverHostname) 126 | { 127 | $this->serverHostname = $serverHostname; 128 | 129 | return $this; 130 | } 131 | 132 | /** 133 | * Client will return random server on each query. 134 | * 135 | * @return $this 136 | */ 137 | public function usingRandomServer() 138 | { 139 | $this->serverHostname = function () { 140 | if ($this->isOnCluster()) { 141 | return $this->serverProvider->getRandomServerFromCluster($this->getClusterName()); 142 | } else { 143 | return $this->serverProvider->getRandomServer(); 144 | } 145 | }; 146 | 147 | return $this; 148 | } 149 | 150 | /** 151 | * Client will use server with tag as server for queries. 152 | * 153 | * @var string 154 | * 155 | * @return $this 156 | */ 157 | public function usingServerWithTag(string $tag) 158 | { 159 | $this->serverHostname = function () use ($tag) { 160 | if ($this->isOnCluster()) { 161 | return $this->serverProvider->getRandomServerFromClusterByTag($this->getClusterName(), $tag); 162 | } else { 163 | return $this->serverProvider->getRandomServerWithTag($tag); 164 | } 165 | }; 166 | 167 | return $this; 168 | } 169 | 170 | /** 171 | * Returns true if cluster selected. 172 | * 173 | * @return bool 174 | */ 175 | protected function isOnCluster(): bool 176 | { 177 | return !is_null($this->getClusterName()); 178 | } 179 | 180 | /** 181 | * Returns server to perform request. 182 | * 183 | * @return Server 184 | */ 185 | public function getServer(): Server 186 | { 187 | if ($this->serverHostname instanceof \Closure) { 188 | $server = call_user_func($this->serverHostname); 189 | } else { 190 | if ($this->isOnCluster()) { 191 | /* 192 | * If no server provided, will take random server from cluster 193 | */ 194 | if (is_null($this->serverHostname)) { 195 | $server = $this->serverProvider->getRandomServerFromCluster($this->getClusterName()); 196 | $this->serverHostname = $server->getHost(); 197 | } else { 198 | $server = $this->serverProvider->getServerFromCluster( 199 | $this->getClusterName(), 200 | $this->serverHostname 201 | ); 202 | } 203 | } else { 204 | /* 205 | * If no server provided, will take random server from cluster 206 | */ 207 | if (is_null($this->serverHostname)) { 208 | $server = $this->serverProvider->getRandomServer(); 209 | $this->serverHostname = $server->getHost(); 210 | } else { 211 | $server = $this->serverProvider->getServer($this->serverHostname); 212 | } 213 | } 214 | } 215 | 216 | return $server; 217 | } 218 | 219 | /** 220 | * Performs select query and returns one result. 221 | * 222 | * Example: 223 | * 224 | * $client->select('select * from table where column = ?', [1]); 225 | * 226 | * @param string $query 227 | * @param FileInterface[] $files 228 | * @param array $settings 229 | * 230 | * @return \Tinderbox\Clickhouse\Query\Result 231 | */ 232 | public function readOne(string $query, array $files = [], array $settings = []): Result 233 | { 234 | $query = $this->createQuery($this->getServer(), $query, $files, $settings); 235 | 236 | $result = $this->getTransport()->read([$query], 1); 237 | 238 | return $result[0]; 239 | } 240 | 241 | /** 242 | * Performs batch of select queries. 243 | * 244 | * @param array $queries 245 | * @param int $concurrency Max concurrency requests 246 | * 247 | * @return array 248 | */ 249 | public function read(array $queries, int $concurrency = 5): array 250 | { 251 | foreach ($queries as $i => $query) { 252 | if (!$query instanceof Query) { 253 | $queries[$i] = $this->guessQuery($query); 254 | } 255 | } 256 | 257 | return $this->getTransport()->read($queries, $concurrency); 258 | } 259 | 260 | /** 261 | * Performs insert or simple statement query. 262 | * 263 | * @param string $query 264 | * @param array $files 265 | * @param array $settings 266 | * 267 | * @return bool 268 | */ 269 | public function writeOne(string $query, array $files = [], array $settings = []): bool 270 | { 271 | if (!$query instanceof Query) { 272 | $query = $this->createQuery($this->getServer(), $query, $files, $settings); 273 | } 274 | 275 | $result = $this->getTransport()->write([$query], 1); 276 | 277 | return $result[0][0]; 278 | } 279 | 280 | /** 281 | * Performs batch of insert or simple statement queries. 282 | * 283 | * @param array $queries 284 | * @param int $concurrency 285 | * 286 | * @return array 287 | */ 288 | public function write(array $queries, int $concurrency = 5): array 289 | { 290 | foreach ($queries as $i => $query) { 291 | if (!$query instanceof Query) { 292 | $queries[$i] = $this->guessQuery($query); 293 | } 294 | } 295 | 296 | return $this->getTransport()->write($queries, $concurrency); 297 | } 298 | 299 | /** 300 | * Performs async insert queries using local csv or tsv files. 301 | * 302 | * @param string $table 303 | * @param array $columns 304 | * @param array $files 305 | * @param string|null $format 306 | * @param array $settings 307 | * @param int $concurrency Max concurrency requests 308 | * 309 | * @return array 310 | */ 311 | public function writeFiles( 312 | string $table, 313 | array $columns, 314 | array $files, 315 | string $format = Format::TSV, 316 | array $settings = [], 317 | int $concurrency = 5 318 | ) { 319 | $sql = 'INSERT INTO '.$table.' ('.implode(', ', $columns).') FORMAT '.strtoupper($format); 320 | 321 | foreach ($files as $i => $file) { 322 | if (!$file instanceof FileInterface) { 323 | $files[$i] = new File($file); 324 | } 325 | } 326 | 327 | $query = $this->createQuery($this->getServer(), $sql, $files, $settings); 328 | 329 | return $this->getTransport()->write([$query], $concurrency); 330 | } 331 | 332 | /** 333 | * Creates query instance from specified arguments. 334 | * 335 | * @param \Tinderbox\Clickhouse\Server $server 336 | * @param string $sql 337 | * @param array $files 338 | * @param array $settings 339 | * 340 | * @return \Tinderbox\Clickhouse\Query 341 | */ 342 | protected function createQuery( 343 | Server $server, 344 | string $sql, 345 | array $files = [], 346 | array $settings = [] 347 | ): Query { 348 | return new Query($server, $sql, $files, $settings); 349 | } 350 | 351 | /** 352 | * Parses query array and returns query instance. 353 | * 354 | * @param array $query 355 | * 356 | * @return \Tinderbox\Clickhouse\Query 357 | */ 358 | protected function guessQuery(array $query): Query 359 | { 360 | $server = $query['server'] ?? $this->getServer(); 361 | $sql = $query['query']; 362 | $tables = $query['files'] ?? []; 363 | $settings = $query['settings'] ?? []; 364 | 365 | return $this->createQuery($server, $sql, $tables, $settings); 366 | } 367 | } 368 | -------------------------------------------------------------------------------- /src/Cluster.php: -------------------------------------------------------------------------------- 1 | name = $name; 44 | $this->addServers($servers); 45 | } 46 | 47 | /** 48 | * Pushes servers to cluster. 49 | * 50 | * @param array $servers Each server can be provided as array or Server instance 51 | * 52 | * @throws \Tinderbox\Clickhouse\Exceptions\ClusterException 53 | * 54 | * @return \Tinderbox\Clickhouse\Cluster 55 | */ 56 | public function addServers(array $servers): self 57 | { 58 | foreach ($servers as $hostname => $server) { 59 | if (!$server instanceof Server && is_array($server)) { 60 | $host = $server['host']; 61 | $port = $server['port'] ?? null; 62 | $database = $server['database'] ?? null; 63 | $username = $server['username'] ?? null; 64 | $password = $server['password'] ?? null; 65 | $options = $server['options'] ?? null; 66 | 67 | $server = new Server($host, $port, $database, $username, $password, $options); 68 | } 69 | 70 | /* @var Server $server */ 71 | if (!is_string($hostname)) { 72 | $hostname = $server->getHost(); 73 | } 74 | 75 | $this->addServer($hostname, $server); 76 | } 77 | 78 | return $this; 79 | } 80 | 81 | /** 82 | * Pushes one server to cluster. 83 | * 84 | * @param string $hostname 85 | * @param \Tinderbox\Clickhouse\Server $server 86 | * 87 | * @throws \Tinderbox\Clickhouse\Exceptions\ClusterException 88 | */ 89 | public function addServer(string $hostname, Server $server) 90 | { 91 | if (isset($this->servers[$hostname])) { 92 | throw ClusterException::serverHostnameDuplicate($hostname); 93 | } 94 | 95 | $this->servers[$hostname] = $server; 96 | 97 | $serverTags = $server->getOptions()->getTags(); 98 | 99 | foreach ($serverTags as $serverTag) { 100 | $this->serversByTags[$serverTag][$hostname] = true; 101 | } 102 | } 103 | 104 | /** 105 | * Returns servers in cluster. 106 | * 107 | * @return \Tinderbox\Clickhouse\Server[] 108 | */ 109 | public function getServers(): array 110 | { 111 | return $this->servers; 112 | } 113 | 114 | /** 115 | * Returns servers in cluster by tag. 116 | * 117 | * @param string $tag 118 | * 119 | * @throws ClusterException 120 | * 121 | * @return \Tinderbox\Clickhouse\Server[] 122 | */ 123 | public function getServersByTag(string $tag): array 124 | { 125 | if (!isset($this->serversByTags[$tag])) { 126 | throw ClusterException::tagNotFound($tag); 127 | } 128 | 129 | return $this->serversByTags[$tag]; 130 | } 131 | 132 | /** 133 | * Returns server by specified hostname. 134 | * 135 | * @param string $hostname 136 | * 137 | * @throws \Tinderbox\Clickhouse\Exceptions\ClusterException 138 | * 139 | * @return \Tinderbox\Clickhouse\Server 140 | */ 141 | public function getServerByHostname(string $hostname): Server 142 | { 143 | if (!isset($this->servers[$hostname])) { 144 | throw ClusterException::serverNotFound($hostname); 145 | } 146 | 147 | return $this->servers[$hostname]; 148 | } 149 | 150 | /** 151 | * Returns cluster name. 152 | * 153 | * @return string 154 | */ 155 | public function getName(): string 156 | { 157 | return $this->name; 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /src/Common/AbstractFile.php: -------------------------------------------------------------------------------- 1 | source = $source; 22 | } 23 | 24 | /** 25 | * Returns full path to source file. 26 | * 27 | * @return string 28 | */ 29 | public function getSource(): string 30 | { 31 | return $this->source; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Common/File.php: -------------------------------------------------------------------------------- 1 | getSource(), 'r'); 14 | 15 | if ($gzip) { 16 | stream_filter_append($handle, 'zlib.deflate', STREAM_FILTER_READ, ['window' => 30]); 17 | } 18 | 19 | return new Stream($handle); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Common/FileFromString.php: -------------------------------------------------------------------------------- 1 | source); 15 | fseek($handle, 0); 16 | 17 | if ($gzip) { 18 | stream_filter_append($handle, 'zlib.deflate', STREAM_FILTER_READ, ['window' => 30]); 19 | } 20 | 21 | return new Stream($handle); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/Common/Format.php: -------------------------------------------------------------------------------- 1 | ['pipe', 'r'], 21 | 1 => ['pipe', 'w'], 22 | 2 => ['pipe', 'w'], 23 | ]; 24 | 25 | $process = proc_open($this->getCcatPath().' '.escapeshellarg($this->source), $descriptorspec, $pipes); 26 | $stream = $pipes[1]; 27 | 28 | if ($gzip) { 29 | stream_filter_append($stream, 'zlib.deflate', STREAM_FILTER_READ, ['window' => 30]); 30 | } 31 | 32 | return new CcatStream(new Stream($stream), $process); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/Common/Protocol.php: -------------------------------------------------------------------------------- 1 | protocol = $protocol; 34 | 35 | return $this; 36 | } 37 | 38 | /** 39 | * Returns protocol. 40 | * 41 | * @return string 42 | */ 43 | public function getProtocol(): string 44 | { 45 | return $this->protocol; 46 | } 47 | 48 | /** 49 | * Set tags. 50 | * 51 | * @param array $tags 52 | * 53 | * @return ServerOptions 54 | */ 55 | public function setTags(array $tags): self 56 | { 57 | $this->tags = []; 58 | 59 | foreach ($tags as $tag) { 60 | $this->addTag($tag); 61 | } 62 | 63 | return $this; 64 | } 65 | 66 | /** 67 | * Adds tag. 68 | * 69 | * @param string $tag 70 | * 71 | * @return ServerOptions 72 | */ 73 | public function addTag(string $tag): self 74 | { 75 | $this->tags[$tag] = true; 76 | 77 | return $this; 78 | } 79 | 80 | /** 81 | * Returns tags. 82 | * 83 | * @return array 84 | */ 85 | public function getTags(): array 86 | { 87 | return array_keys($this->tags); 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/Common/TempTable.php: -------------------------------------------------------------------------------- 1 | name = $name; 52 | $this->structure = $structure; 53 | $this->format = $format; 54 | 55 | $this->setSource($source); 56 | } 57 | 58 | protected function setSource($source) 59 | { 60 | if (is_scalar($source)) { 61 | $source = new File($source); 62 | } 63 | 64 | $this->source = $source; 65 | } 66 | 67 | /** 68 | * Returns table name. 69 | * 70 | * @return string 71 | */ 72 | public function getName(): string 73 | { 74 | return $this->name; 75 | } 76 | 77 | /** 78 | * Returns table structure. 79 | * 80 | * @return array 81 | */ 82 | public function getStructure(): array 83 | { 84 | return $this->structure; 85 | } 86 | 87 | /** 88 | * Returns format. 89 | * 90 | * @return string 91 | */ 92 | public function getFormat(): string 93 | { 94 | return $this->format; 95 | } 96 | 97 | public function open(bool $gzip = true): StreamInterface 98 | { 99 | return $this->source->open($gzip); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/Exceptions/ClientException.php: -------------------------------------------------------------------------------- 1 | getHost().':'.$server->getPort().'] with error: ['.$reasonMessage.']'); 17 | } 18 | 19 | public static function serverReturnedError($exception, Query $query) 20 | { 21 | if ($exception instanceof RequestException) { 22 | $error = $exception->getResponse()->getBody()->getContents(); 23 | } else { 24 | $error = $exception; 25 | } 26 | 27 | $errorString = 'Host ['.$query->getServer()->getHost().'] returned error: '.$error.'. Query: '.$query->getQuery(); 28 | 29 | return new static($errorString); 30 | } 31 | 32 | public static function malformedResponseFromServer($response) 33 | { 34 | return new static('Malformed response from server: '.$response); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/Interfaces/FileInterface.php: -------------------------------------------------------------------------------- 1 | server = $server; 49 | $this->query = $query; 50 | $this->files = $files; 51 | $this->settings = $settings; 52 | } 53 | 54 | /** 55 | * Returns SQL query. 56 | * 57 | * @return string 58 | */ 59 | public function getQuery(): string 60 | { 61 | return $this->query; 62 | } 63 | 64 | /** 65 | * Returns files attached to query. 66 | * 67 | * @return array 68 | */ 69 | public function getFiles(): array 70 | { 71 | return $this->files; 72 | } 73 | 74 | /** 75 | * Returns server to process query. 76 | * 77 | * @return \Tinderbox\Clickhouse\Server 78 | */ 79 | public function getServer(): Server 80 | { 81 | return $this->server; 82 | } 83 | 84 | /** 85 | * Returns settings. 86 | * 87 | * @return array 88 | */ 89 | public function getSettings(): array 90 | { 91 | return $this->settings; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/Query/QueryStatistic.php: -------------------------------------------------------------------------------- 1 | rows = $rows; 60 | $this->bytes = $bytes; 61 | $this->time = $time; 62 | $this->rowsBeforeLimitAtLeast = $rowsBeforeLimitAtLeast; 63 | } 64 | 65 | /** 66 | * Returns number of read rows. 67 | * 68 | * @return int 69 | */ 70 | public function getRows(): int 71 | { 72 | return $this->rows; 73 | } 74 | 75 | /** 76 | * Returns number of read bytes. 77 | * 78 | * @return int 79 | */ 80 | public function getBytes(): int 81 | { 82 | return $this->bytes; 83 | } 84 | 85 | /** 86 | * Returns query execution time. 87 | * 88 | * @return float 89 | */ 90 | public function getTime(): float 91 | { 92 | return $this->time; 93 | } 94 | 95 | /** 96 | * Returns rows before limit at least. 97 | * 98 | * @return int|null 99 | */ 100 | public function getRowsBeforeLimitAtLeast(): ?int 101 | { 102 | return $this->rowsBeforeLimitAtLeast; 103 | } 104 | 105 | /** 106 | * Getter to simplify access to rows, bytes and time. 107 | * 108 | * @param string $name 109 | * 110 | * @throws \Tinderbox\Clickhouse\Exceptions\QueryStatisticException 111 | * 112 | * @return mixed 113 | */ 114 | public function __get($name) 115 | { 116 | $method = 'get'.ucfirst($name); 117 | 118 | if (method_exists($this, $method)) { 119 | return call_user_func([$this, $method]); 120 | } 121 | 122 | throw QueryStatisticException::propertyNotExists($name); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/Query/Result.php: -------------------------------------------------------------------------------- 1 | setQuery($query); 57 | $this->setRows($rows); 58 | $this->setStatistic($statistic); 59 | } 60 | 61 | /** 62 | * Sets query. 63 | * 64 | * @param Query $query 65 | */ 66 | protected function setQuery(Query $query) 67 | { 68 | $this->query = $query; 69 | } 70 | 71 | /** 72 | * Sets statistic. 73 | * 74 | * @param \Tinderbox\Clickhouse\Query\QueryStatistic $statistic 75 | */ 76 | protected function setStatistic(QueryStatistic $statistic) 77 | { 78 | $this->statistic = $statistic; 79 | } 80 | 81 | /** 82 | * Sets rows. 83 | * 84 | * @param array $rows 85 | */ 86 | protected function setRows(array $rows) 87 | { 88 | $this->rows = $rows; 89 | } 90 | 91 | /** 92 | * Returns rows. 93 | * 94 | * @return array 95 | */ 96 | public function getRows(): array 97 | { 98 | return $this->rows; 99 | } 100 | 101 | /** 102 | * Returns query. 103 | * 104 | * @return Query 105 | */ 106 | public function getQuery(): Query 107 | { 108 | return $this->query; 109 | } 110 | 111 | /** 112 | * Returns statistic. 113 | * 114 | * @return \Tinderbox\Clickhouse\Query\QueryStatistic 115 | */ 116 | public function getStatistic(): QueryStatistic 117 | { 118 | return $this->statistic; 119 | } 120 | 121 | /** 122 | * Getter to simplify access to rows and statistic. 123 | * 124 | * @param string $name 125 | * 126 | * @throws \Tinderbox\Clickhouse\Exceptions\ResultException 127 | * 128 | * @return mixed 129 | */ 130 | public function __get($name) 131 | { 132 | $method = 'get'.ucfirst($name); 133 | 134 | if (method_exists($this, $method)) { 135 | return call_user_func([$this, $method]); 136 | } 137 | 138 | throw ResultException::propertyNotExists($name); 139 | } 140 | 141 | /* 142 | * ArrayAccess 143 | */ 144 | 145 | public function offsetExists($offset) 146 | { 147 | return isset($this->rows[$offset]); 148 | } 149 | 150 | public function offsetGet($offset) 151 | { 152 | return $this->rows[$offset]; 153 | } 154 | 155 | public function offsetSet($offset, $value) 156 | { 157 | throw ResultException::isReadonly(); 158 | } 159 | 160 | public function offsetUnset($offset) 161 | { 162 | throw ResultException::isReadonly(); 163 | } 164 | 165 | /* 166 | * Iterator 167 | */ 168 | 169 | public function current() 170 | { 171 | return $this->rows[$this->current]; 172 | } 173 | 174 | public function next() 175 | { 176 | $this->current++; 177 | } 178 | 179 | public function key() 180 | { 181 | return $this->current; 182 | } 183 | 184 | public function valid() 185 | { 186 | return isset($this->rows[$this->current]); 187 | } 188 | 189 | public function rewind() 190 | { 191 | $this->current = 0; 192 | } 193 | 194 | /* 195 | * Countable 196 | */ 197 | 198 | public function count() 199 | { 200 | return count($this->rows); 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /src/Server.php: -------------------------------------------------------------------------------- 1 | setHost($host); 73 | $this->setPort($port); 74 | $this->setDatabase($database); 75 | $this->setUsername($username); 76 | $this->setPassword($password); 77 | $this->setOptions($options); 78 | } 79 | 80 | /** 81 | * Sets host. 82 | * 83 | * @param string $host 84 | * 85 | * @return \Tinderbox\Clickhouse\Server 86 | */ 87 | public function setHost(string $host): self 88 | { 89 | $this->host = $host; 90 | 91 | return $this; 92 | } 93 | 94 | /** 95 | * Returns host. 96 | * 97 | * @return string 98 | */ 99 | public function getHost(): string 100 | { 101 | return $this->host; 102 | } 103 | 104 | /** 105 | * Sets port. 106 | * 107 | * @param string $port 108 | * 109 | * @return \Tinderbox\Clickhouse\Server 110 | */ 111 | public function setPort(string $port): self 112 | { 113 | $this->port = $port; 114 | 115 | return $this; 116 | } 117 | 118 | /** 119 | * Returns port. 120 | * 121 | * @return string 122 | */ 123 | public function getPort(): string 124 | { 125 | return $this->port; 126 | } 127 | 128 | /** 129 | * Sets database. 130 | * 131 | * @param string|null $database 132 | * 133 | * @return \Tinderbox\Clickhouse\Server 134 | */ 135 | public function setDatabase(string $database = null): self 136 | { 137 | $this->database = $database; 138 | 139 | return $this; 140 | } 141 | 142 | /** 143 | * Returns database. 144 | * 145 | * @return null|string 146 | */ 147 | public function getDatabase(): ?string 148 | { 149 | return $this->database; 150 | } 151 | 152 | /** 153 | * Sets username. 154 | * 155 | * @param string|null $username 156 | * 157 | * @return \Tinderbox\Clickhouse\Server 158 | */ 159 | public function setUsername(string $username = null): self 160 | { 161 | $this->username = $username; 162 | 163 | return $this; 164 | } 165 | 166 | /** 167 | * Returns username. 168 | * 169 | * @return null|string 170 | */ 171 | public function getUsername(): ?string 172 | { 173 | return $this->username; 174 | } 175 | 176 | /** 177 | * Sets password. 178 | * 179 | * @param string|null $password 180 | * 181 | * @return \Tinderbox\Clickhouse\Server 182 | */ 183 | public function setPassword(string $password = null): self 184 | { 185 | $this->password = $password; 186 | 187 | return $this; 188 | } 189 | 190 | /** 191 | * Returns password. 192 | * 193 | * @return null|string 194 | */ 195 | public function getPassword(): ?string 196 | { 197 | return $this->password; 198 | } 199 | 200 | /** 201 | * Sets options. 202 | * 203 | * If no options provided, will use default options 204 | * 205 | * @param \Tinderbox\Clickhouse\Common\ServerOptions|null $options 206 | * 207 | * @return \Tinderbox\Clickhouse\Server 208 | */ 209 | public function setOptions(ServerOptions $options = null): self 210 | { 211 | if (is_null($options)) { 212 | return $this->setDefaultOptions(); 213 | } 214 | 215 | $this->options = $options; 216 | 217 | return $this; 218 | } 219 | 220 | /** 221 | * Sets default options. 222 | * 223 | * @return \Tinderbox\Clickhouse\Server 224 | */ 225 | protected function setDefaultOptions(): self 226 | { 227 | $this->options = new ServerOptions(); 228 | 229 | return $this; 230 | } 231 | 232 | /** 233 | * Returns options. 234 | * 235 | * @return \Tinderbox\Clickhouse\Common\ServerOptions 236 | */ 237 | public function getOptions(): ServerOptions 238 | { 239 | return $this->options; 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /src/ServerProvider.php: -------------------------------------------------------------------------------- 1 | getName(); 50 | 51 | if (isset($this->clusters[$clusterName])) { 52 | throw ServerProviderException::clusterExists($clusterName); 53 | } 54 | 55 | $this->clusters[$clusterName] = $cluster; 56 | 57 | return $this; 58 | } 59 | 60 | public function addServer(Server $server) 61 | { 62 | $serverHostname = $server->getHost(); 63 | 64 | if (isset($this->servers[$serverHostname])) { 65 | throw ServerProviderException::serverHostnameDuplicate($serverHostname); 66 | } 67 | 68 | $this->servers[$serverHostname] = $server; 69 | 70 | $serverTags = $server->getOptions()->getTags(); 71 | 72 | foreach ($serverTags as $serverTag) { 73 | $this->serversByTags[$serverTag][$serverHostname] = true; 74 | } 75 | 76 | return $this; 77 | } 78 | 79 | public function getRandomServer(): Server 80 | { 81 | return $this->getServer(array_rand($this->servers, 1)); 82 | } 83 | 84 | public function getRandomServerWithTag(string $tag): Server 85 | { 86 | if (!isset($this->serversByTags[$tag])) { 87 | throw ServerProviderException::serverTagNotFound($tag); 88 | } 89 | 90 | return $this->getServer(array_rand($this->serversByTags[$tag], 1)); 91 | } 92 | 93 | public function getRandomServerFromCluster(string $cluster): Server 94 | { 95 | $cluster = $this->getCluster($cluster); 96 | $randomServerHostname = array_rand($cluster->getServers(), 1); 97 | 98 | return $cluster->getServerByHostname($randomServerHostname); 99 | } 100 | 101 | public function getRandomServerFromClusterByTag(string $cluster, string $tag): Server 102 | { 103 | $cluster = $this->getCluster($cluster); 104 | 105 | $randomServerHostname = array_rand($cluster->getServersByTag($tag), 1); 106 | 107 | return $cluster->getServerByHostname($randomServerHostname); 108 | } 109 | 110 | public function getServerFromCluster(string $cluster, string $serverHostname) 111 | { 112 | $cluster = $this->getCluster($cluster); 113 | 114 | return $cluster->getServerByHostname($serverHostname); 115 | } 116 | 117 | public function getServer(string $serverHostname): Server 118 | { 119 | if (!isset($this->servers[$serverHostname])) { 120 | throw ServerProviderException::serverHostnameNotFound($serverHostname); 121 | } 122 | 123 | return $this->servers[$serverHostname]; 124 | } 125 | 126 | public function getCluster(string $cluster): Cluster 127 | { 128 | if (!isset($this->clusters[$cluster])) { 129 | throw ServerProviderException::clusterNotFound($cluster); 130 | } 131 | 132 | return $this->clusters[$cluster]; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/Support/CcatStream.php: -------------------------------------------------------------------------------- 1 | process = $process; 23 | } 24 | 25 | public function getSize() 26 | { 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/Transport/HttpTransport.php: -------------------------------------------------------------------------------- 1 | [ 37 | * 'timeout' => 50, 38 | * 'connect_timeout => 10, 39 | * ], 40 | * 'write' => [ 41 | * 'debug' => true, 42 | * 'timeout' => 100, 43 | * ], 44 | * 'deflate' => true 45 | * ] 46 | * 47 | * @var array 48 | */ 49 | private $options; 50 | 51 | /** 52 | * HttpTransport constructor. 53 | * 54 | * @param Client $client 55 | * @param array $options 56 | */ 57 | public function __construct(Client $client = null, array $options = []) 58 | { 59 | $this->setClient($client); 60 | 61 | $this->options = $options; 62 | } 63 | 64 | /** 65 | * Returns flag to enable / disable queries and data compression. 66 | * 67 | * @return bool 68 | */ 69 | protected function isDeflateEnabled(): bool 70 | { 71 | return $this->options['deflate'] ?? true; 72 | } 73 | 74 | /** 75 | * Returns default headers for requests. 76 | * 77 | * @return array 78 | */ 79 | protected function getHeaders() 80 | { 81 | $headers = [ 82 | 'Accept-Encoding' => 'gzip', 83 | ]; 84 | 85 | if ($this->isDeflateEnabled()) { 86 | $headers['Content-Encoding'] = 'gzip'; 87 | } 88 | 89 | return $headers; 90 | } 91 | 92 | /** 93 | * Sets Guzzle client. 94 | * 95 | * @param Client|null $client 96 | */ 97 | protected function setClient(Client $client = null) 98 | { 99 | if (is_null($client)) { 100 | $this->httpClient = $this->createHttpClient(); 101 | } else { 102 | $this->httpClient = $client; 103 | } 104 | } 105 | 106 | /** 107 | * Creates Guzzle client. 108 | */ 109 | protected function createHttpClient() 110 | { 111 | return new Client(); 112 | } 113 | 114 | /** 115 | * Executes write queries. 116 | * 117 | * @param array $queries 118 | * @param int $concurrency 119 | * 120 | * @throws \Throwable 121 | * 122 | * @return array 123 | */ 124 | public function write(array $queries, int $concurrency = 5): array 125 | { 126 | $result = []; 127 | $openedStreams = []; 128 | 129 | foreach ($queries as $query) { 130 | $requests = function (Query $query) use (&$openedStreams) { 131 | if (!empty($query->getFiles())) { 132 | foreach ($query->getFiles() as $file) { 133 | /* @var FileInterface $file */ 134 | $headers = $this->getHeaders(); 135 | 136 | $uri = $this->buildRequestUri($query->getServer(), [ 137 | 'query' => $query->getQuery(), 138 | ], $query->getSettings()); 139 | 140 | $stream = $file->open($this->isDeflateEnabled()); 141 | $openedStreams[] = $stream; 142 | 143 | $request = new Request('POST', $uri, $headers, $stream); 144 | 145 | yield $request; 146 | } 147 | } else { 148 | $headers = $this->getHeaders(); 149 | 150 | $uri = $this->buildRequestUri($query->getServer(), [], $query->getSettings()); 151 | 152 | $sql = $this->isDeflateEnabled() ? gzencode($query->getQuery()) : $query->getQuery(); 153 | $request = new Request('POST', $uri, $headers, $sql); 154 | 155 | yield $request; 156 | } 157 | }; 158 | 159 | $queryResult = []; 160 | 161 | $pool = new Pool( 162 | $this->httpClient, 163 | $requests($query), 164 | [ 165 | 'concurrency' => $concurrency, 166 | 'fulfilled' => function ($response, $index) use (&$queryResult) { 167 | $queryResult[$index] = true; 168 | }, 169 | 'rejected' => $this->parseReason($query), 170 | 'options' => array_merge([ 171 | 'expect' => false, 172 | ], $this->options['write'] ?? []), 173 | ] 174 | ); 175 | 176 | $promise = $pool->promise(); 177 | 178 | try { 179 | $promise->wait(); 180 | } catch (\Throwable $exception) { 181 | foreach ($openedStreams as $openedStream) { 182 | $openedStream->close(); 183 | } 184 | 185 | throw $exception; 186 | } 187 | 188 | ksort($result); 189 | 190 | foreach ($openedStreams as $openedStream) { 191 | $openedStream->close(); 192 | } 193 | 194 | $result[] = $queryResult; 195 | } 196 | 197 | return $result; 198 | } 199 | 200 | public function read(array $queries, int $concurrency = 5): array 201 | { 202 | $openedStreams = []; 203 | 204 | $requests = function ($queries) use (&$openedStreams) { 205 | foreach ($queries as $index => $query) { 206 | /* @var Query $query */ 207 | 208 | $params = [ 209 | 'wait_end_of_query' => 1, 210 | ]; 211 | 212 | $multipart = [ 213 | [ 214 | 'name' => 'query', 215 | 'contents' => $query->getQuery().' FORMAT JSON', 216 | ], 217 | ]; 218 | 219 | foreach ($query->getFiles() as $file) { 220 | /* @var TempTable $file */ 221 | $tableQueryParams = $this->getTempTableQueryParams($file); 222 | 223 | $stream = $file->open(false); 224 | $openedStreams[] = $stream; 225 | 226 | $multipart[] = [ 227 | 'name' => $file->getName(), 228 | 'contents' => $stream, 229 | 'filename' => $file->getName(), 230 | ]; 231 | 232 | $params = array_merge($tableQueryParams, $params); 233 | } 234 | 235 | $body = new MultipartStream($multipart); 236 | 237 | $uri = $this->buildRequestUri($query->getServer(), $params, $query->getSettings()); 238 | 239 | yield $index => new Request('POST', $uri, [], $body); 240 | } 241 | }; 242 | 243 | $result = []; 244 | 245 | $pool = new Pool( 246 | $this->httpClient, 247 | $requests($queries), 248 | [ 249 | 'concurrency' => $concurrency, 250 | 'fulfilled' => function ($response, $index) use (&$result, $queries) { 251 | $result[$index] = $this->assembleResult($queries[$index], $response); 252 | }, 253 | 'rejected' => function ($response, $index) use ($queries) { 254 | $query = $queries[$index]; 255 | 256 | $this->parseReason($query)($response); 257 | }, 258 | 'options' => array_merge([ 259 | 'expect' => false, 260 | ], $this->options['read'] ?? []), 261 | ] 262 | ); 263 | 264 | $promise = $pool->promise(); 265 | 266 | try { 267 | $promise->wait(); 268 | } catch (\Throwable $exception) { 269 | foreach ($openedStreams as $openedStream) { 270 | $openedStream->close(); 271 | } 272 | 273 | throw $exception; 274 | } 275 | 276 | ksort($result); 277 | 278 | foreach ($openedStreams as $openedStream) { 279 | $openedStream->close(); 280 | } 281 | 282 | return $result; 283 | } 284 | 285 | /** 286 | * Parse temp table data to append it to request. 287 | * 288 | * @param \Tinderbox\Clickhouse\Common\TempTable $table 289 | * 290 | * @return array 291 | */ 292 | protected function getTempTableQueryParams(TempTable $table) 293 | { 294 | list($structure, $withColumns) = $this->assembleTempTableStructure($table); 295 | 296 | return [ 297 | $table->getName().'_'.($withColumns ? 'structure' : 'types') => $structure, 298 | $table->getName().'_format' => $table->getFormat(), 299 | ]; 300 | } 301 | 302 | /** 303 | * Assembles string from TempTable structure. 304 | * 305 | * @param \Tinderbox\Clickhouse\Common\TempTable $table 306 | * 307 | * @return string 308 | */ 309 | protected function assembleTempTableStructure(TempTable $table) 310 | { 311 | $structure = $table->getStructure(); 312 | $withColumns = true; 313 | 314 | $preparedStructure = []; 315 | 316 | foreach ($structure as $column => $type) { 317 | if (is_int($column)) { 318 | $withColumns = false; 319 | $preparedStructure[] = $type; 320 | } else { 321 | $preparedStructure[] = $column.' '.$type; 322 | } 323 | } 324 | 325 | return [implode(', ', $preparedStructure), $withColumns]; 326 | } 327 | 328 | /** 329 | * Determines the reason why request was rejected. 330 | * 331 | * @param Query $query 332 | * 333 | * @return \Closure 334 | */ 335 | protected function parseReason(Query $query) 336 | { 337 | return function ($reason) use ($query) { 338 | if ($reason instanceof RequestException) { 339 | $response = $reason->getResponse(); 340 | 341 | if (is_null($response)) { 342 | throw TransportException::connectionError($query->getServer(), $reason->getMessage()); 343 | } else { 344 | throw TransportException::serverReturnedError($reason, $query); 345 | } 346 | } 347 | 348 | throw $reason; 349 | }; 350 | } 351 | 352 | /** 353 | * Assembles Result instance from server response. 354 | * 355 | * @param Query $query 356 | * @param \Psr\Http\Message\ResponseInterface $response 357 | * 358 | * @return \Tinderbox\Clickhouse\Query\Result 359 | */ 360 | protected function assembleResult(Query $query, ResponseInterface $response): Result 361 | { 362 | $response = $response->getBody()->getContents(); 363 | 364 | try { 365 | $result = \GuzzleHttp\json_decode($response, true); 366 | 367 | $statistic = new QueryStatistic( 368 | $result['statistics']['rows_read'] ?? 0, 369 | $result['statistics']['bytes_read'] ?? 0, 370 | $result['statistics']['elapsed'] ?? 0, 371 | $result['rows_before_limit_at_least'] ?? null 372 | ); 373 | 374 | return new Result($query, $result['data'] ?? [], $statistic); 375 | } catch (\Exception $e) { 376 | throw TransportException::malformedResponseFromServer($response); 377 | } 378 | } 379 | 380 | /** 381 | * Builds uri with necessary params. 382 | * 383 | * @param \Tinderbox\Clickhouse\Server $server 384 | * @param array $query 385 | * @param array $settings 386 | * 387 | * @return string 388 | */ 389 | protected function buildRequestUri(Server $server, array $query = [], array $settings = []): string 390 | { 391 | $uri = $server->getOptions()->getProtocol().'://'.$server->getHost().':'.$server->getPort(); 392 | 393 | if (!is_null($server->getDatabase())) { 394 | $query['database'] = $server->getDatabase(); 395 | } 396 | 397 | if (!is_null($server->getUsername())) { 398 | $query['user'] = $server->getUsername(); 399 | } 400 | 401 | if (!is_null($server->getPassword())) { 402 | $query['password'] = $server->getPassword(); 403 | } 404 | 405 | $query = array_merge($query, $settings); 406 | 407 | return $uri.'?'.http_build_query($query); 408 | } 409 | } 410 | -------------------------------------------------------------------------------- /tests/CcatStreamTest.php: -------------------------------------------------------------------------------- 1 | assertNull($ccatStream->getSize()); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/ClientTest.php: -------------------------------------------------------------------------------- 1 | addServer($server); 28 | 29 | $client = new Client($serverProvider); 30 | 31 | $this->assertEquals( 32 | $serverProvider->getRandomServer(), 33 | $client->getServer(), 34 | 'Correctly passes server provider' 35 | ); 36 | } 37 | 38 | public function testTransports() 39 | { 40 | $server = new Server('127.0.0.1'); 41 | $serverProvider = new ServerProvider(); 42 | $serverProvider->addServer($server); 43 | 44 | $transport = $this->createMock(TransportInterface::class); 45 | $transport->method('read')->willReturn([ 46 | new Result(new Query($server, ''), [0, 1], new QueryStatistic(0, 0, 0, 0)), 47 | ]); 48 | 49 | $client = new Client($serverProvider, $transport); 50 | 51 | $result = $client->readOne('test query'); 52 | 53 | $this->assertEquals(2, count($result->rows), 'Correctly changes transport'); 54 | } 55 | 56 | public function testClusters() 57 | { 58 | $cluster = new Cluster( 59 | 'test', 60 | [ 61 | new Server('127.0.0.1'), 62 | new Server('127.0.0.2'), 63 | new Server('127.0.0.3'), 64 | ] 65 | ); 66 | 67 | $cluster2 = new Cluster( 68 | 'test2', 69 | [ 70 | new Server('127.0.0.4'), 71 | new Server('127.0.0.5'), 72 | new Server('127.0.0.6'), 73 | ] 74 | ); 75 | 76 | $serverProvider = new ServerProvider(); 77 | $serverProvider->addCluster($cluster)->addCluster($cluster2); 78 | 79 | $client = new Client($serverProvider); 80 | 81 | $server = $client->onCluster('test')->getServer(); /* will return random server from cluster */ 82 | $this->assertContains( 83 | $server, 84 | $cluster->getServers(), 85 | 'Correctly returns random server from specified cluster' 86 | ); 87 | 88 | $this->assertEquals($server, $client->getServer(), 'Remembers firstly selected random server for next calls'); 89 | 90 | $client->using('127.0.0.3'); 91 | $server = $client->getServer(); 92 | 93 | $this->assertEquals( 94 | '127.0.0.3', 95 | $server->getHost(), 96 | 'Correctly returns specified server from specified cluster' 97 | ); 98 | 99 | $server = $client->onCluster('test2')->getServer(); /* will return random server from cluster */ 100 | $this->assertContains( 101 | $server, 102 | $cluster2->getServers(), 103 | 'Correctly returns random server from specified cluster' 104 | ); 105 | 106 | $client->usingRandomServer(); 107 | $server = $client->getServer(); 108 | 109 | while ($server === $client->getServer()) { 110 | /* Randomize while get non used server */ 111 | } 112 | 113 | $this->assertTrue(true, 'Correctly randomizes cluster servers on each call'); 114 | 115 | $this->expectException(ClusterException::class); 116 | $this->expectExceptionMessage('Server with hostname [127.0.0.0] is not found in cluster'); 117 | 118 | $client->onCluster('test')->using('127.0.0.0')->getServer(); 119 | } 120 | 121 | public function testServers() 122 | { 123 | $server1 = new Server('127.0.0.1'); 124 | $server2 = new Server('127.0.0.2'); 125 | $server3 = new Server('127.0.0.3'); 126 | 127 | $serverProvider = new ServerProvider(); 128 | $serverProvider->addServer($server1)->addServer($server2)->addServer($server3); 129 | 130 | $client = new Client($serverProvider); 131 | 132 | $server = $client->getServer(); 133 | $this->assertTrue( 134 | in_array($server->getHost(), ['127.0.0.1', '127.0.0.2', '127.0.0.3'], true), 135 | 'Correctly returns random server' 136 | ); 137 | 138 | $this->assertEquals($server, $client->getServer(), 'Remembers firstly selected random server for next calls'); 139 | 140 | $server = $client->using('127.0.0.3')->getServer(); 141 | $this->assertEquals('127.0.0.3', $server->getHost(), 'Correctly returns specified server'); 142 | 143 | $client->usingRandomServer(); 144 | $server = $client->getServer(); 145 | 146 | while ($server === $client->getServer()) { 147 | /* Randomize while get non used server */ 148 | } 149 | 150 | $this->assertTrue(true, 'Correctly randomizes cluster servers on each call'); 151 | 152 | $this->expectException(ServerProviderException::class); 153 | $this->expectExceptionMessage('Can not find server with hostname [127.0.0.0]'); 154 | 155 | $client->using('127.0.0.0')->getServer(); 156 | } 157 | 158 | public function testServersWithTags() 159 | { 160 | $serverOptionsWithTag = (new ServerOptions())->addTag('tag'); 161 | 162 | $serverWithTag = (new Server('127.0.0.1', 8123))->setOptions($serverOptionsWithTag); 163 | $serverWithoutTag = new Server('127.0.0.2', 8123); 164 | 165 | $serverProvider = new ServerProvider(); 166 | $serverProvider->addServer($serverWithTag)->addServer($serverWithoutTag); 167 | 168 | $client = new Client($serverProvider); 169 | $client->usingServerWithTag('tag'); 170 | 171 | $server = $client->getServer(); 172 | 173 | $this->assertEquals('127.0.0.1', $server->getHost()); 174 | $this->assertEquals(8123, $server->getPort()); 175 | } 176 | 177 | public function testServersWithTagsOnCluster() 178 | { 179 | $serverOptionsWithTag = (new ServerOptions())->addTag('tag'); 180 | 181 | $serverWithTag = (new Server('127.0.0.1', 8123))->setOptions($serverOptionsWithTag); 182 | $serverWithoutTag = new Server('127.0.0.2', 8123); 183 | 184 | $cluster = new Cluster( 185 | 'test', 186 | [ 187 | $serverWithTag, 188 | $serverWithoutTag, 189 | ] 190 | ); 191 | 192 | $serverProvider = new ServerProvider(); 193 | $serverProvider->addCluster($cluster); 194 | 195 | $client = new Client($serverProvider); 196 | $client->onCluster('test')->usingServerWithTag('tag'); 197 | 198 | $server = $client->getServer(); 199 | 200 | $this->assertEquals('127.0.0.1', $server->getHost()); 201 | $this->assertEquals(8123, $server->getPort()); 202 | } 203 | 204 | public function testClusterAndServersTogether() 205 | { 206 | $cluster = new Cluster( 207 | 'test', 208 | [ 209 | new Server('127.0.0.1'), 210 | new Server('127.0.0.2'), 211 | new Server('127.0.0.3'), 212 | ] 213 | ); 214 | 215 | $server1 = new Server('127.0.0.4'); 216 | $server2 = new Server('127.0.0.5'); 217 | $server3 = new Server('127.0.0.6'); 218 | 219 | $serverProvider = new ServerProvider(); 220 | $serverProvider->addCluster($cluster)->addServer($server1)->addServer($server2)->addServer($server3); 221 | 222 | $client = new Client($serverProvider); 223 | 224 | $server = $client->getServer(); 225 | $this->assertTrue( 226 | in_array($server->getHost(), ['127.0.0.4', '127.0.0.5', '127.0.0.6'], true), 227 | 'Correctly returns random server not in cluster' 228 | ); 229 | 230 | $this->assertEquals($server, $client->getServer(), 'Remembers firstly selected random server for next calls'); 231 | 232 | $client->onCluster('test'); 233 | 234 | $server = $client->onCluster('test')->getServer(); /* will return random server from cluster */ 235 | $this->assertContains( 236 | $server, 237 | $cluster->getServers(), 238 | 'Correctly returns random server from specified cluster' 239 | ); 240 | 241 | $this->assertEquals($server, $client->getServer(), 'Remembers firstly selected random server for next calls'); 242 | 243 | $server = $client->onCluster(null)->getServer(); 244 | $this->assertTrue( 245 | in_array($server->getHost(), ['127.0.0.4', '127.0.0.5', '127.0.0.6'], true), 246 | 'Correctly returns random server after disabling cluster mode' 247 | ); 248 | } 249 | 250 | protected function getClient(): Client 251 | { 252 | $serverProvider = new ServerProvider(); 253 | $serverProvider->addServer(new Server('127.0.0.1', '8123', 'default', 'default', '')); 254 | 255 | return new Client($serverProvider); 256 | } 257 | 258 | public function testReadOne() 259 | { 260 | $client = $this->getClient(); 261 | 262 | $result = $client->readOne('select * from numbers(0, 10) order by number desc'); 263 | 264 | $this->assertEquals(10, count($result->rows), 'Correctly executes query using mapper'); 265 | } 266 | 267 | public function testRead() 268 | { 269 | $client = $this->getClient(); 270 | 271 | $result = $client->read( 272 | [ 273 | [ 274 | 'query' => 'select * from numbers(0, 10) order by number desc', 275 | ], 276 | new Query($client->getServer(), 'select * from numbers(0, 20) order by number desc'), 277 | new Query($client->getServer(), 'select * from numbers(0, 20) where number in tab order by number desc', [ 278 | new TempTable('tab', new FileFromString('1'.PHP_EOL.'2'.PHP_EOL), ['number' => 'UInt64'], Format::TSV), 279 | ]), 280 | ] 281 | ); 282 | 283 | $this->assertEquals(10, count($result[0]->rows), 'Correctly converts query from array to query instance'); 284 | $this->assertEquals(20, count($result[1]->rows), 'Correctly executes queries'); 285 | $this->assertEquals(2, count($result[2]->rows), 'Correctly executes query with file'); 286 | } 287 | 288 | public function testWrite() 289 | { 290 | $client = $this->getClient(); 291 | 292 | $client->write([ 293 | new Query($client->getServer(), 'drop table if exists default.tdchc_test_table'), 294 | new Query($client->getServer(), 'create table default.tdchc_test_table (number UInt64) engine = Memory'), 295 | ], 1); 296 | 297 | $client->writeOne('insert into default.tdchc_test_table (number) FORMAT TSV', [ 298 | new FileFromString('1'.PHP_EOL.'2'.PHP_EOL), 299 | ]); 300 | 301 | $result = $client->readOne('select * from default.tdchc_test_table'); 302 | 303 | $this->assertEquals(2, count($result->rows), 'Correctly writes data'); 304 | } 305 | 306 | public function testWriteFiles() 307 | { 308 | $client = $this->getClient(); 309 | 310 | $client->write([ 311 | new Query($client->getServer(), 'drop table if exists default.tdchc_test_table'), 312 | new Query($client->getServer(), 'create table default.tdchc_test_table (number UInt64) engine = Memory'), 313 | ], 1); 314 | 315 | $client->writeFiles('default.tdchc_test_table', ['number'], [ 316 | new FileFromString('1'.PHP_EOL.'2'.PHP_EOL), 317 | new FileFromString('3'.PHP_EOL.'4'.PHP_EOL), 318 | new FileFromString('5'.PHP_EOL.'6'.PHP_EOL), 319 | ], Format::TSV); 320 | 321 | $result = $client->readOne('select * from default.tdchc_test_table'); 322 | 323 | $this->assertEquals(6, count($result->rows), 'Correctly writes data'); 324 | } 325 | } 326 | -------------------------------------------------------------------------------- /tests/ClusterTest.php: -------------------------------------------------------------------------------- 1 | assertEquals('test_cluster', $cluster->getName(), 'Return correct cluster name passed in constructor'); 26 | $this->assertEquals($servers[0], $cluster->getServers()[$servers[0]->getHost()], 'Return correct cluster structure'); 27 | $this->assertEquals($servers[1], $cluster->getServers()[$servers[1]->getHost()], 'Return correct cluster structure'); 28 | 29 | $this->assertEquals($servers[0], $cluster->getServerByHostname('127.0.0.1'), 'Return correct server by hostname'); 30 | $this->assertEquals($servers[1], $cluster->getServerByHostname('127.0.0.2', 'Return correct server by hostname')); 31 | 32 | $this->expectException(ClusterException::class); 33 | $this->expectExceptionMessage('Server with hostname [unknown_hostname] is not found in cluster'); 34 | $cluster->getServerByHostname('unknown_hostname'); 35 | } 36 | 37 | public function testServersWithAliases() 38 | { 39 | $servers = [ 40 | new Server('127.0.0.1'), 41 | 'aliased' => new Server('127.0.0.2'), 42 | ]; 43 | 44 | $cluster = new Cluster('test_cluster', $servers); 45 | 46 | $this->assertEquals($servers[0], $cluster->getServers()[$servers[0]->getHost()], 'Return correct cluster structure'); 47 | $this->assertEquals($servers['aliased'], $cluster->getServers()['aliased'], 'Return correct cluster structure'); 48 | 49 | $this->assertEquals($servers[0], $cluster->getServerByHostname('127.0.0.1'), 'Return correct server by hostname'); 50 | $this->assertEquals($servers['aliased'], $cluster->getServerByHostname('aliased', 'Return correct server by hostname')); 51 | } 52 | 53 | public function testServerFromArray() 54 | { 55 | $server = [ 56 | 'host' => '127.0.0.2', 57 | 'port' => '123', 58 | 'database' => 'not_default', 59 | 'username' => 'user', 60 | 'password' => 'secret', 61 | 'options' => (new ServerOptions())->setProtocol('https'), 62 | ]; 63 | 64 | $cluster = new Cluster('test_cluster', [ 65 | $server, 66 | ]); 67 | 68 | $createdServer = $cluster->getServerByHostname('127.0.0.2'); 69 | 70 | $this->assertEquals($server['host'], $createdServer->getHost(), 'Correctly passes server host to server constructor from array'); 71 | $this->assertEquals($server['port'], $createdServer->getPort(), 'Correctly passes server port to server constructor from array'); 72 | $this->assertEquals($server['database'], $createdServer->getDatabase(), 'Correctly passes server database to server constructor from array'); 73 | $this->assertEquals($server['username'], $createdServer->getUsername(), 'Correctly passes server username to server constructor from array'); 74 | $this->assertEquals($server['password'], $createdServer->getPassword(), 'Correctly passes server password to server constructor from array'); 75 | $this->assertEquals($server['options'], $createdServer->getOptions(), 'Correctly passes server options to server constructor from array'); 76 | } 77 | 78 | public function testServersDuplicates() 79 | { 80 | $servers = [ 81 | new Server('127.0.0.1'), 82 | new Server('127.0.0.1'), 83 | ]; 84 | 85 | $this->expectException(ClusterException::class); 86 | $this->expectExceptionMessage('Hostname [127.0.0.1] already provided'); 87 | 88 | new Cluster('test_cluster', $servers); 89 | } 90 | 91 | public function testServersWithTags() 92 | { 93 | $server = [ 94 | 'host' => '127.0.0.1', 95 | 'port' => 8123, 96 | 'database' => 'default', 97 | 'username' => 'default', 98 | 'password' => '', 99 | 'options' => (new ServerOptions())->addTag('tag'), 100 | ]; 101 | 102 | $cluster = new Cluster('test', [ 103 | $server, 104 | ]); 105 | 106 | $servers = $cluster->getServersByTag('tag'); 107 | 108 | $this->assertEquals(array_keys($servers)[0], $server['host']); 109 | } 110 | 111 | public function testServerTagNotFound() 112 | { 113 | $cluster = new Cluster('test', []); 114 | 115 | $this->expectException(ClusterException::class); 116 | $this->expectExceptionMessage('There are no servers with tag [tag] in cluster'); 117 | 118 | $cluster->getServersByTag('tag'); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /tests/FileFromStringTest.php: -------------------------------------------------------------------------------- 1 | open(false); 26 | 27 | $this->assertEquals($result, $stream->getContents(), 'Correctly reads content from file without encoding'); 28 | } 29 | 30 | public function testFileWithGzip() 31 | { 32 | $fileContent = []; 33 | 34 | for ($i = 0; $i < 100; $i++) { 35 | $fileContent[] = $i.PHP_EOL; 36 | } 37 | 38 | $result = implode('', $fileContent); 39 | 40 | $file = new FileFromString($result); 41 | $stream = $file->open(); 42 | 43 | $this->assertEquals($result, gzdecode($stream->getContents()), 'Correctly reads content from file with encoding'); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /tests/FileTest.php: -------------------------------------------------------------------------------- 1 | open(false); 32 | 33 | $this->assertEquals($result, $stream->getContents(), 'Correctly reads content from file without encoding'); 34 | 35 | unlink($fileName); 36 | } 37 | 38 | public function testFileWithGzip() 39 | { 40 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 41 | $fileContent = []; 42 | $result = []; 43 | 44 | for ($i = 0; $i < 100; $i++) { 45 | $fileContent[] = $i.PHP_EOL; 46 | 47 | $result[] = $i.PHP_EOL; 48 | } 49 | 50 | $result = implode('', $result); 51 | 52 | file_put_contents($fileName, implode('', $fileContent)); 53 | 54 | $file = new File($fileName); 55 | $stream = $file->open(); 56 | 57 | $this->assertEquals($result, gzdecode($stream->getContents()), 'Correctly reads content from file with encoding'); 58 | 59 | unlink($fileName); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /tests/HttpTransportTest.php: -------------------------------------------------------------------------------- 1 | $handler, 30 | ])); 31 | } 32 | 33 | protected function getQuery(): Query 34 | { 35 | return new Query($this->getServer(), 'select * from table'); 36 | } 37 | 38 | protected function getServer(): Server 39 | { 40 | return new Server(CLICKHOUSE_SERVER_HOST, CLICKHOUSE_SERVER_PORT, 'default', 'default'); 41 | } 42 | 43 | protected function getTransport(): HttpTransport 44 | { 45 | return new HttpTransport(); 46 | } 47 | 48 | public function testRead() 49 | { 50 | $transport = $this->getMockedTransport([ 51 | new Response(200, [], json_encode([ 52 | 'data' => [ 53 | [ 54 | '1' => 1, 55 | ], 56 | ], 57 | 'statistics' => [ 58 | 'rows_read' => 1, 59 | 'bytes_read' => 1, 60 | 'elapsed' => 0.100, 61 | ], 62 | 'rows_before_limit_at_least' => 1024, 63 | ])), 64 | ]); 65 | 66 | $result = $transport->read([$this->getQuery()]); 67 | 68 | $this->assertEquals([ 69 | [ 70 | '1' => 1, 71 | ], 72 | ], $result[0]->rows, 'Returns correct response from server'); 73 | 74 | $this->assertEquals([ 75 | 'rows_read' => 1, 76 | 'bytes_read' => 1, 77 | 'elapsed' => 0.100, 78 | ], [ 79 | 'rows_read' => $result[0]->statistic->rows, 80 | 'bytes_read' => $result[0]->statistic->bytes, 81 | 'elapsed' => $result[0]->statistic->time, 82 | ], 'Returns correct statistic from server'); 83 | 84 | $this->assertEquals(1024, $result[0]->statistic->rowsBeforeLimitAtLeast, 'Returns correct rows_before_limit_at_least'); 85 | } 86 | 87 | public function testReadMultipleRequests() 88 | { 89 | $transport = $this->getMockedTransport([ 90 | new Response(200, [], json_encode([ 91 | 'data' => [ 92 | [ 93 | '1' => 1, 94 | ], 95 | ], 96 | 'statistics' => [ 97 | 'rows_read' => 1, 98 | 'bytes_read' => 1, 99 | 'elapsed' => 0.100, 100 | ], 101 | 'rows_before_limit_at_least' => 1024, 102 | ])), 103 | 104 | new Response(200, [], json_encode([ 105 | 'data' => [ 106 | [ 107 | '1' => 2, 108 | ], 109 | ], 110 | 'statistics' => [ 111 | 'rows_read' => 2, 112 | 'bytes_read' => 2, 113 | 'elapsed' => 0.100, 114 | ], 115 | 'rows_before_limit_at_least' => 1025, 116 | ])), 117 | ]); 118 | 119 | $result = $transport->read([$this->getQuery(), $this->getQuery()]); 120 | 121 | $this->assertEquals([ 122 | [ 123 | '1' => 1, 124 | ], 125 | ], $result[0]->rows, 'Returns correct response from server'); 126 | 127 | $this->assertEquals([ 128 | 'rows_read' => 1, 129 | 'bytes_read' => 1, 130 | 'elapsed' => 0.100, 131 | ], [ 132 | 'rows_read' => $result[0]->statistic->rows, 133 | 'bytes_read' => $result[0]->statistic->bytes, 134 | 'elapsed' => $result[0]->statistic->time, 135 | ], 'Returns correct statistic from server'); 136 | 137 | $this->assertEquals(1024, $result[0]->statistic->rowsBeforeLimitAtLeast, 'Returns correct rows_before_limit_at_least'); 138 | 139 | $this->assertEquals([ 140 | [ 141 | '1' => 1, 142 | ], 143 | ], $result[0]->rows, 'Returns correct response from server'); 144 | 145 | $this->assertEquals([ 146 | 'rows_read' => 2, 147 | 'bytes_read' => 2, 148 | 'elapsed' => 0.100, 149 | ], [ 150 | 'rows_read' => $result[1]->statistic->rows, 151 | 'bytes_read' => $result[1]->statistic->bytes, 152 | 'elapsed' => $result[1]->statistic->time, 153 | ], 'Returns correct statistic from server'); 154 | 155 | $this->assertEquals(1025, $result[1]->statistic->rowsBeforeLimitAtLeast, 'Returns correct rows_before_limit_at_least'); 156 | } 157 | 158 | public function testReadWithTablesUnstructured() 159 | { 160 | $transport = $this->getTransport(); 161 | 162 | $tableSource = $this->getTempFileName(); 163 | 164 | file_put_contents($tableSource, '1'.PHP_EOL.'2'); 165 | 166 | $table = new TempTable('temp', $tableSource, ['UInt64'], Format::TSV); 167 | 168 | $query = new Query($this->getServer(), 'select * from numbers(0, 10) where number in temp', [$table]); 169 | $result = $transport->read([$query]); 170 | 171 | $this->assertEquals([1, 2], array_column($result[0]->rows, 'number')); 172 | 173 | unlink($tableSource); 174 | } 175 | 176 | public function testReadWithTablesStructured() 177 | { 178 | $transport = $this->getTransport(); 179 | 180 | $tableSource = $this->getTempFileName(); 181 | 182 | file_put_contents($tableSource, '1'.PHP_EOL.'2'); 183 | 184 | $table = new TempTable('temp', $tableSource, ['number' => 'UInt64'], Format::TSV); 185 | 186 | $query = new Query($this->getServer(), 'select number from temp', [$table]); 187 | $result = $transport->read([$query]); 188 | 189 | $this->assertEquals([1, 2], array_column($result[0]->rows, 'number'), 'Returns correct result when uses temp tables for read queries'); 190 | 191 | unlink($tableSource); 192 | } 193 | 194 | public function testReadWithTablesReject() 195 | { 196 | $transport = $this->getTransport(); 197 | 198 | $query = new Query($this->getServer(), 'select * from temp2'); 199 | 200 | $this->expectException(TransportException::class); 201 | 202 | $transport->read([$query]); 203 | } 204 | 205 | public function testWrite() 206 | { 207 | $transport = $this->getTransport(); 208 | 209 | $query = new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'); 210 | $result = $transport->write([$query]); 211 | 212 | $this->assertTrue($result[0][0], 'Returns true on write queries without files'); 213 | 214 | $query = new Query($this->getServer(), 'create table default.tdchc_test_table (number UInt64, string String) engine = Memory'); 215 | $transport->write([$query]); 216 | 217 | $tableSource = $this->getTempFileName(); 218 | 219 | file_put_contents($tableSource, "1\tsome".PHP_EOL."2\tstring"); 220 | 221 | $table = new File($tableSource); 222 | 223 | $query = new Query($this->getServer(), 'insert into default.tdchc_test_table (number, string) FORMAT TSV', [$table]); 224 | $result = $transport->write([$query]); 225 | 226 | $this->assertTrue($result[0][0], 'Returns true on write queries'); 227 | 228 | $query = new Query($this->getServer(), 'select * from default.tdchc_test_table order by number'); 229 | $result = $transport->read([$query]); 230 | 231 | $this->assertEquals([ 232 | [ 233 | 'number' => 1, 234 | 'string' => 'some', 235 | ], 236 | [ 237 | 'number' => 2, 238 | 'string' => 'string', 239 | ], 240 | ], $result[0]->rows, 'Returns correct result from recently created table and filled with temp files'); 241 | 242 | $handle = $table->open(); 243 | 244 | $query = new Query($this->getServer(), 'insert into default.tdchc_test_table (number, string) FORMAT TSV', [$table]); 245 | $result = $transport->write([$query]); 246 | 247 | $this->assertNotEquals($handle, $table->open(), 'Correctly closes file stream after write query'); 248 | 249 | $query = new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'); 250 | $transport->write([$query]); 251 | 252 | unlink($tableSource); 253 | } 254 | 255 | public function testWriteMultipleFilesPerOneQuery() 256 | { 257 | $transport = $this->getTransport(); 258 | 259 | $transport->write([ 260 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 261 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table_2'), 262 | ]); 263 | 264 | $transport->write([ 265 | new Query($this->getServer(), 'create table default.tdchc_test_table (number UInt64, string String) engine = Memory'), 266 | new Query($this->getServer(), 'create table default.tdchc_test_table_2 (number UInt64, string String) engine = Memory'), 267 | ]); 268 | 269 | $tableSource = $this->getTempFileName(); 270 | $tableSource2 = $this->getTempFileName(); 271 | 272 | file_put_contents($tableSource, "1\tsome".PHP_EOL."2\tstring"); 273 | file_put_contents($tableSource2, "3\tstring".PHP_EOL."4\tsome"); 274 | 275 | $table = new File($tableSource); 276 | $table2 = new File($tableSource2); 277 | 278 | $result = $transport->write([ 279 | new Query($this->getServer(), 'insert into default.tdchc_test_table (number, string) FORMAT TSV', [$table, $table2]), 280 | new Query($this->getServer(), 'insert into default.tdchc_test_table_2 (number, string) FORMAT TSV', [$table2]), 281 | ]); 282 | 283 | $this->assertTrue($result[0][0] && $result[0][1] && $result[1][0], 'Returns true on multiple write queries with multiple files'); 284 | 285 | $result = $transport->read([ 286 | new Query($this->getServer(), 'select * from default.tdchc_test_table order by number'), 287 | new Query($this->getServer(), 'select * from default.tdchc_test_table_2 order by number '), 288 | ]); 289 | 290 | $this->assertEquals([ 291 | [ 292 | 'number' => 1, 293 | 'string' => 'some', 294 | ], 295 | [ 296 | 'number' => 2, 297 | 'string' => 'string', 298 | ], 299 | [ 300 | 'number' => 3, 301 | 'string' => 'string', 302 | ], 303 | [ 304 | 'number' => 4, 305 | 'string' => 'some', 306 | ], 307 | ], $result[0]->rows, 'Returns correct result from recently created table and filled with temp files'); 308 | 309 | $this->assertEquals([ 310 | [ 311 | 'number' => 3, 312 | 'string' => 'string', 313 | ], 314 | [ 315 | 'number' => 4, 316 | 'string' => 'some', 317 | ], 318 | ], $result[1]->rows, 'Returns correct result from recently created table and filled with temp files'); 319 | 320 | $transport->write([ 321 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 322 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table_2'), 323 | ]); 324 | 325 | unlink($tableSource); 326 | unlink($tableSource2); 327 | } 328 | 329 | protected function getTempFileName(): string 330 | { 331 | return tempnam(sys_get_temp_dir(), 'tbchc_'); 332 | } 333 | 334 | public function testConnectionError() 335 | { 336 | $transport = new HttpTransport(null, ['read' => ['connect_timeout' => 0.1]]); 337 | 338 | $this->expectException(TransportException::class); 339 | $this->expectExceptionMessage('Can\'t connect to the server [KHGIUYhakljsfnk:8123]'); 340 | 341 | $transport->read([ 342 | new Query(new Server('KHGIUYhakljsfnk'), ''), 343 | ]); 344 | } 345 | 346 | public function testUnknownReason() 347 | { 348 | $transport = $this->getMockedTransport([ 349 | new \Exception('Unknown exception'), 350 | ]); 351 | 352 | $this->expectException(\Exception::class); 353 | $this->expectExceptionMessage('Unknown exception'); 354 | 355 | $transport->read([ 356 | new Query($this->getServer(), ''), 357 | ]); 358 | } 359 | 360 | public function testHttpTransportMalformedResponse() 361 | { 362 | $transport = $this->getMockedTransport([ 363 | new Response(200, [], 'some text'), 364 | ]); 365 | 366 | $e = TransportException::malformedResponseFromServer('some text'); 367 | $this->expectException(TransportException::class); 368 | $this->expectExceptionMessage($e->getMessage()); 369 | 370 | $transport->read([ 371 | new Query($this->getServer(), ''), 372 | ]); 373 | } 374 | 375 | public function testConnectionWithPassword() 376 | { 377 | $transport = $this->getTransport(); 378 | 379 | $this->expectException(TransportException::class); 380 | $regularExpression = '/Authentication failed: password is incorrect/'; 381 | if (method_exists($this, 'expectExceptionMessageRegExp')) { 382 | $this->expectExceptionMessageRegExp($regularExpression); 383 | } else { 384 | $this->expectErrorMessageMatches($regularExpression); 385 | } 386 | 387 | $transport->read([ 388 | new Query(new Server('127.0.0.1', 8123, 'default', 'default', 'pass'), 'select 1', [ 389 | new TempTable('name', new FileFromString('aaa'), ['string' => 'String'], Format::TSV), 390 | ]), 391 | ]); 392 | } 393 | 394 | public function testConnectionWithPasswordOnWrite() 395 | { 396 | $transport = $this->getTransport(); 397 | 398 | $this->expectException(TransportException::class); 399 | $regularExpression = '/Authentication failed: password is incorrect/'; 400 | if (method_exists($this, 'expectExceptionMessageRegExp')) { 401 | $this->expectExceptionMessageRegExp($regularExpression); 402 | } else { 403 | $this->expectErrorMessageMatches($regularExpression); 404 | } 405 | 406 | $transport->write([ 407 | new Query(new Server('127.0.0.1', 8123, 'default', 'default', 'pass'), 'insert into table 1', [ 408 | new FileFromString('aaa'), 409 | ]), 410 | ]); 411 | } 412 | 413 | public function testFileInsert() 414 | { 415 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 416 | $fileContent = []; 417 | 418 | for ($i = 0; $i < 100; $i++) { 419 | $fileContent[] = $i.PHP_EOL; 420 | } 421 | 422 | file_put_contents($fileName, implode('', $fileContent)); 423 | 424 | $file = new File($fileName); 425 | 426 | $transport = $this->getTransport(); 427 | 428 | $transport->write([ 429 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 430 | ]); 431 | 432 | $transport->write([ 433 | new Query($this->getServer(), 'create table default.tdchc_test_table (number UInt64) engine = MergeTree order by number'), 434 | ]); 435 | 436 | $transport->write([ 437 | new Query($this->getServer(), 'insert into default.tdchc_test_table (number) FORMAT TSV', [$file]), 438 | ]); 439 | 440 | $result = $transport->read([ 441 | new Query($this->getServer(), 'select count() from default.tdchc_test_table'), 442 | ]); 443 | 444 | $this->assertEquals(100, $result[0]->rows[0]['count()'], 'File content may be used in insert statements'); 445 | 446 | $transport->write([ 447 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 448 | ]); 449 | } 450 | 451 | public function testMergedFilesInsert() 452 | { 453 | $ccatFileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 454 | $ccatFileContent = []; 455 | 456 | for ($i = 0; $i < 100; $i++) { 457 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 458 | file_put_contents($fileName, $i.PHP_EOL); 459 | 460 | $ccatFileContent[] = $fileName; 461 | } 462 | 463 | file_put_contents($ccatFileName, implode(PHP_EOL, $ccatFileContent)); 464 | 465 | $file = new MergedFiles($ccatFileName); 466 | 467 | $transport = $this->getTransport(); 468 | 469 | $transport->write([ 470 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 471 | ]); 472 | 473 | $transport->write([ 474 | new Query($this->getServer(), 'create table default.tdchc_test_table (number UInt64) engine = MergeTree order by number'), 475 | ]); 476 | 477 | $transport->write([ 478 | new Query($this->getServer(), 'insert into default.tdchc_test_table (number) FORMAT TSV', [$file]), 479 | ]); 480 | 481 | $result = $transport->read([ 482 | new Query($this->getServer(), 'select count() from default.tdchc_test_table'), 483 | ]); 484 | 485 | $this->assertEquals(100, $result[0]->rows[0]['count()'], 'File content may be used in insert statements'); 486 | 487 | $transport->write([ 488 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 489 | ]); 490 | } 491 | 492 | public function testTempTableReadAndWrite() 493 | { 494 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 495 | $fileContent = []; 496 | 497 | for ($i = 0; $i < 100; $i++) { 498 | $fileContent[] = $i."\t".($i >= 50 ? 'string' : 'some').PHP_EOL; 499 | } 500 | 501 | file_put_contents($fileName, implode('', $fileContent)); 502 | 503 | $file = new TempTable('name', $fileName, ['number' => 'UInt64', 'string' => 'String'], Format::TSV); 504 | 505 | $transport = $this->getTransport(); 506 | 507 | $transport->write([ 508 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 509 | ]); 510 | 511 | $transport->write([ 512 | new Query($this->getServer(), 'create table default.tdchc_test_table (number UInt64, string String) engine = MergeTree order by number'), 513 | ]); 514 | 515 | $transport->write([ 516 | new Query($this->getServer(), 'insert into default.tdchc_test_table (number, string) FORMAT TSV', [$file]), 517 | ]); 518 | 519 | $result = $transport->read([ 520 | new Query($this->getServer(), 'select string, count() from default.tdchc_test_table group by string'), 521 | ]); 522 | 523 | $this->assertTrue($result[0]->rows[0]['count()'] == 50 && $result[0]->rows[1]['count()'] == 50, 'File content may be used in insert statements'); 524 | 525 | $result = $transport->read([ 526 | new Query($this->getServer(), 'select string, count() from name group by string', [$file]), 527 | ]); 528 | 529 | $this->assertTrue($result[0]->rows[0]['count()'] == 50 && $result[0]->rows[1]['count()'] == 50, 'File content may be used in read statements'); 530 | 531 | $transport->write([ 532 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 533 | ]); 534 | } 535 | 536 | public function testFileFromStringReadAndWrite() 537 | { 538 | $fileContent = []; 539 | 540 | for ($i = 0; $i < 100; $i++) { 541 | $fileContent[] = $i."\t".($i >= 50 ? 'string' : 'some').PHP_EOL; 542 | } 543 | 544 | $file = new FileFromString(implode('', $fileContent)); 545 | $table = new TempTable('name', $file, ['number' => 'UInt64', 'string' => 'String'], Format::TSV); 546 | 547 | $transport = $this->getTransport(); 548 | 549 | $transport->write([ 550 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 551 | new Query($this->getServer(), 'create table default.tdchc_test_table (number UInt64, string String) engine = MergeTree order by number'), 552 | new Query($this->getServer(), 'insert into default.tdchc_test_table (number, string) FORMAT TSV', [$file]), 553 | ]); 554 | 555 | $result = $transport->read([ 556 | new Query($this->getServer(), 'select string, count() from default.tdchc_test_table group by string'), 557 | new Query($this->getServer(), 'select string, count() from name group by string', [$table]), 558 | ]); 559 | 560 | $this->assertTrue($result[0]->rows[0]['count()'] == 50 && $result[0]->rows[1]['count()'] == 50, 'File content may be used in insert statements'); 561 | $this->assertTrue($result[1]->rows[0]['count()'] == 50 && $result[1]->rows[1]['count()'] == 50, 'File content may be used in read statements'); 562 | 563 | $transport->write([ 564 | new Query($this->getServer(), 'drop table if exists default.tdchc_test_table'), 565 | ]); 566 | } 567 | } 568 | -------------------------------------------------------------------------------- /tests/MergeFilesStreamTest.php: -------------------------------------------------------------------------------- 1 | open(false); 35 | 36 | $this->assertEquals($result, $stream->getContents(), 'Correctly reads content from ccat without encoding'); 37 | 38 | unlink($ccatFileName); 39 | 40 | foreach ($ccatFileContent as $ccatFile) { 41 | unlink($ccatFile); 42 | } 43 | } 44 | 45 | public function testFileWithGzip() 46 | { 47 | $ccatFileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 48 | $ccatFileContent = []; 49 | $result = []; 50 | 51 | for ($i = 0; $i < 100; $i++) { 52 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 53 | file_put_contents($fileName, $i.PHP_EOL); 54 | 55 | $ccatFileContent[] = $fileName; 56 | 57 | $result[] = $i.PHP_EOL; 58 | } 59 | 60 | $result = implode('', $result); 61 | 62 | file_put_contents($ccatFileName, implode(PHP_EOL, $ccatFileContent)); 63 | 64 | $file = new MergedFiles($ccatFileName); 65 | $stream = $file->open(); 66 | 67 | $this->assertEquals($result, gzdecode($stream->getContents()), 'Correctly reads content from ccat with encoding'); 68 | 69 | unlink($ccatFileName); 70 | 71 | foreach ($ccatFileContent as $ccatFile) { 72 | unlink($ccatFile); 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /tests/QueryStatisticTest.php: -------------------------------------------------------------------------------- 1 | assertEquals(100, $statistic->getRows(), 'Returns correct rows number passed in constructor'); 19 | $this->assertEquals(1024, $statistic->getBytes(), 'Returns correct bytes passed in constructor'); 20 | $this->assertEquals(0.122, $statistic->getTime(), 'Returns correct time passed in constructor'); 21 | $this->assertEquals(1024, $statistic->getRowsBeforeLimitAtLeast(), 'Returns correct rows before limit at least passed in constructor'); 22 | 23 | $this->assertEquals(100, $statistic->rows, 'Returns correct rows number passed in constructor via magic property'); 24 | $this->assertEquals(1024, $statistic->bytes, 'Returns correct bytes passed in constructor via magic property'); 25 | $this->assertEquals(0.122, $statistic->time, 'Returns correct time passed in constructor via magic property'); 26 | $this->assertEquals(1024, $statistic->rowsBeforeLimitAtLeast, 'Returns correct rows before limit at least passed in constructor via magic property'); 27 | 28 | $e = QueryStatisticException::propertyNotExists('miss'); 29 | $this->expectException(QueryStatisticException::class); 30 | $this->expectExceptionMessage($e->getMessage()); 31 | 32 | $statistic->miss; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /tests/QueryTest.php: -------------------------------------------------------------------------------- 1 | 'value', 22 | ]; 23 | 24 | $query = new Query($server, 'select * from table', $files, $settings); 25 | 26 | $this->assertEquals($server, $query->getServer(), 'Returns correct server passed to constructor'); 27 | $this->assertEquals('select * from table', $query->getQuery(), 'Returns correct query passed to constructor'); 28 | $this->assertEquals($files, $query->getFiles(), 'Returns correct files passed to constructor'); 29 | $this->assertEquals($settings, $query->getSettings(), 'Returns correct settings passed to constructor'); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tests/ResultTest.php: -------------------------------------------------------------------------------- 1 | 1, 22 | ], 23 | [ 24 | 'col' => 2, 25 | ], 26 | [ 27 | 'col' => 3, 28 | ], 29 | [ 30 | 'col' => 4, 31 | ], 32 | [ 33 | 'col' => 5, 34 | ], 35 | ]; 36 | $query = new Query(new Server('localhost'), ''); 37 | $statistic = new QueryStatistic(5, 1024, 0.122); 38 | 39 | $result = new Result($query, $rows, $statistic); 40 | 41 | $this->assertEquals($rows, $result->getRows(), 'Returns rows passed to constructor'); 42 | $this->assertEquals($statistic, $result->getStatistic(), 'Returns statistic passed to constructor'); 43 | $this->assertEquals($query, $result->getQuery(), 'Returns query passed to constructor'); 44 | 45 | $this->assertEquals($rows, $result->rows, 'Returns rows passed to constructor via magic property'); 46 | $this->assertEquals($statistic, $result->statistic, 'Returns statistic passed to constructor via magic property'); 47 | $this->assertEquals($query, $result->query, 'Returns query passed to constructor via magic property'); 48 | 49 | $e = ResultException::propertyNotExists('miss'); 50 | $this->expectException(ResultException::class); 51 | $this->expectExceptionMessage($e->getMessage()); 52 | 53 | $result->miss; 54 | } 55 | 56 | public function testResultCountable() 57 | { 58 | $query = new Query(new Server('localhost'), ''); 59 | $statistic = new QueryStatistic(5, 1024, 0.122); 60 | 61 | $result = new Result($query, ['', '', ''], $statistic); 62 | 63 | $this->assertEquals(3, count($result), 'Returns correct rows count via Countable interface'); 64 | } 65 | 66 | public function testResultArrayAccessSet() 67 | { 68 | $rows = [ 69 | [ 70 | 'col' => 1, 71 | ], 72 | [ 73 | 'col' => 2, 74 | ], 75 | [ 76 | 'col' => 3, 77 | ], 78 | [ 79 | 'col' => 4, 80 | ], 81 | [ 82 | 'col' => 5, 83 | ], 84 | ]; 85 | $query = new Query(new Server('localhost'), ''); 86 | $statistic = new QueryStatistic(5, 1024, 0.122); 87 | 88 | $result = new Result($query, $rows, $statistic); 89 | 90 | $e = ResultException::isReadonly(); 91 | $this->expectException(ResultException::class); 92 | $this->expectExceptionMessage($e->getMessage()); 93 | 94 | $result[1] = 'test'; 95 | } 96 | 97 | public function testResultArrayAccessGet() 98 | { 99 | $rows = [ 100 | [ 101 | 'col' => 1, 102 | ], 103 | [ 104 | 'col' => 2, 105 | ], 106 | [ 107 | 'col' => 3, 108 | ], 109 | [ 110 | 'col' => 4, 111 | ], 112 | [ 113 | 'col' => 5, 114 | ], 115 | ]; 116 | $query = new Query(new Server('localhost'), ''); 117 | $statistic = new QueryStatistic(5, 1024, 0.122); 118 | 119 | $result = new Result($query, $rows, $statistic); 120 | $this->assertTrue(isset($result[1]), 'Correctly determines that offset exists via ArrayAccess interface'); 121 | $this->assertFalse(isset($result[10]), 'Correctly determines that offset does not exists via ArrayAccess interface'); 122 | $this->assertEquals($rows[0], $result[0], 'Correctly returns offset via ArrayAccess interface'); 123 | } 124 | 125 | public function testResultArrayAccessUnset() 126 | { 127 | $rows = [ 128 | [ 129 | 'col' => 1, 130 | ], 131 | [ 132 | 'col' => 2, 133 | ], 134 | [ 135 | 'col' => 3, 136 | ], 137 | [ 138 | 'col' => 4, 139 | ], 140 | [ 141 | 'col' => 5, 142 | ], 143 | ]; 144 | $query = new Query(new Server('localhost'), ''); 145 | $statistic = new QueryStatistic(5, 1024, 0.122); 146 | 147 | $result = new Result($query, $rows, $statistic); 148 | 149 | $e = ResultException::isReadonly(); 150 | $this->expectException(ResultException::class); 151 | $this->expectExceptionMessage($e->getMessage()); 152 | 153 | unset($result[1]); 154 | } 155 | 156 | public function testResultIterator() 157 | { 158 | $this->setName('Correctly iterates over rows via Iterator interface'); 159 | 160 | $rows = [ 161 | [ 162 | 'col' => 1, 163 | ], 164 | [ 165 | 'col' => 2, 166 | ], 167 | [ 168 | 'col' => 3, 169 | ], 170 | [ 171 | 'col' => 4, 172 | ], 173 | [ 174 | 'col' => 5, 175 | ], 176 | ]; 177 | $query = new Query(new Server('localhost'), ''); 178 | $statistic = new QueryStatistic(5, 1024, 0.122); 179 | 180 | $result = new Result($query, $rows, $statistic); 181 | 182 | $prev = null; 183 | 184 | foreach ($result as $i => $row) { 185 | $this->assertNotEquals($prev, $row); 186 | 187 | $prev = $row; 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /tests/SanitizerTest.php: -------------------------------------------------------------------------------- 1 | assertEquals($value, $escaped); 19 | } 20 | 21 | public function testEscapeStringValue() 22 | { 23 | $value = "some-test with 'quotes'"; 24 | $escaped = Sanitizer::escape($value); 25 | 26 | $this->assertEquals("'some-test with \'quotes\''", $escaped); 27 | 28 | $value = 'some-test with \'quotes\''; 29 | $escaped = Sanitizer::escape($value); 30 | 31 | $this->assertEquals("'some-test with \'quotes\''", $escaped); 32 | 33 | $value = 'some-test with / \slashes'; 34 | $escaped = Sanitizer::escape($value); 35 | 36 | $this->assertEquals("'some-test with / \\\\slashes'", $escaped); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tests/ServerOptionsTest.php: -------------------------------------------------------------------------------- 1 | assertEquals('http', $options->getProtocol(), 'Sets correct default protocol'); 18 | 19 | $options->setProtocol('https'); 20 | 21 | $this->assertEquals('https', $options->getProtocol(), 'Sets correct protocol'); 22 | } 23 | 24 | public function testTagsFromServerOptions() 25 | { 26 | $options = new ServerOptions(); 27 | 28 | $options->addTag('test'); 29 | 30 | $this->assertTrue( 31 | in_array('test', $options->getTags(), true), 32 | 'Sets correct tags' 33 | ); 34 | 35 | $options->addTag('tag'); 36 | 37 | $this->assertTrue( 38 | in_array('test', $options->getTags(), true), 39 | 'Sets correct tags' 40 | ); 41 | $this->assertTrue( 42 | in_array('tag', $options->getTags(), true), 43 | 'Sets correct tags' 44 | ); 45 | 46 | $options->setTags(['other']); 47 | 48 | $this->assertTrue( 49 | in_array('other', $options->getTags(), true), 50 | 'Sets correct tags' 51 | ); 52 | 53 | $options->addTag('tag'); 54 | 55 | $this->assertTrue( 56 | in_array('other', $options->getTags(), true), 57 | 'Sets correct tags' 58 | ); 59 | $this->assertTrue( 60 | in_array('tag', $options->getTags(), true), 61 | 'Sets correct tags' 62 | ); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /tests/ServerProviderTest.php: -------------------------------------------------------------------------------- 1 | addCluster($cluster); 26 | 27 | $this->assertEquals($cluster, $provider->getCluster('test_cluster'), 'Correctly adds cluster'); 28 | 29 | $firstServer = $provider->getRandomServerFromCluster('test_cluster'); 30 | 31 | while ($firstServer == $provider->getRandomServerFromCluster('test_cluster')) { 32 | /* Randomize while get non used server */ 33 | } 34 | 35 | $this->assertTrue(true, 'Correctly randomizes cluster servers'); 36 | 37 | $this->assertEquals($servers[1], $provider->getServerFromCluster('test_cluster', '127.0.0.2'), 'Correctly returns server from cluster by hostname'); 38 | } 39 | 40 | public function testClusterDuplicate() 41 | { 42 | $servers = [ 43 | new Server('127.0.0.1'), 44 | new Server('127.0.0.2'), 45 | ]; 46 | 47 | $cluster = new Cluster('test_cluster', $servers); 48 | 49 | $provider = new ServerProvider(); 50 | $provider->addCluster($cluster); 51 | 52 | $this->expectException(ServerProviderException::class); 53 | $this->expectExceptionMessage('Can not add cluster with name [test_cluster], because it already added'); 54 | 55 | $provider->addCluster($cluster); 56 | } 57 | 58 | public function testClusterNotFound() 59 | { 60 | $provider = new ServerProvider(); 61 | 62 | $this->expectException(ServerProviderException::class); 63 | $this->expectExceptionMessage('Can not find cluster with name [test_cluster]'); 64 | 65 | $provider->getCluster('test_cluster'); 66 | } 67 | 68 | public function testServers() 69 | { 70 | $servers = [ 71 | new Server('127.0.0.1'), 72 | new Server('127.0.0.2'), 73 | ]; 74 | 75 | $provider = new ServerProvider(); 76 | $provider->addServer($servers[0]); 77 | $provider->addServer($servers[1]); 78 | 79 | $this->assertEquals($servers[0], $provider->getServer('127.0.0.1'), 'Correctly adds server and returns it by hostname'); 80 | 81 | $firstServer = $provider->getRandomServer(); 82 | 83 | while ($firstServer === $provider->getRandomServer()) { 84 | /* Randomize while get non used server */ 85 | } 86 | 87 | $this->assertTrue(true, 'Correctly randomizes servers'); 88 | } 89 | 90 | public function testServerDuplicate() 91 | { 92 | $server = new Server('127.0.0.1'); 93 | 94 | $provider = new ServerProvider(); 95 | $provider->addServer($server); 96 | 97 | $this->expectException(ServerProviderException::class); 98 | $this->expectExceptionMessage('Server with hostname [127.0.0.1] already provided'); 99 | 100 | $provider->addServer($server); 101 | } 102 | 103 | public function testServerNotFound() 104 | { 105 | $provider = new ServerProvider(); 106 | 107 | $this->expectException(ServerProviderException::class); 108 | $this->expectExceptionMessage('Can not find server with hostname [127.0.0.1]'); 109 | 110 | $provider->getServer('127.0.0.1'); 111 | } 112 | 113 | public function testServersWithTags() 114 | { 115 | $serverOptionsWithTag = (new ServerOptions())->addTag('tag'); 116 | 117 | $serverWithTag = new Server('127.0.0.1', 8123, 'default', 'default', '', $serverOptionsWithTag); 118 | $serverWithoutTag = new Server('127.0.0.2', 8123); 119 | 120 | $provider = new ServerProvider(); 121 | $provider->addServer($serverWithTag); 122 | $provider->addServer($serverWithoutTag); 123 | 124 | $server = $provider->getRandomServerWithTag('tag'); 125 | $this->assertEquals($server->getHost(), $serverWithTag->getHost(), 'Correctly adds server with tag and returns it'); 126 | } 127 | 128 | public function testServerTagNotFound() 129 | { 130 | $provider = new ServerProvider(); 131 | 132 | $this->expectException(ServerProviderException::class); 133 | $this->expectExceptionMessage('Can not find servers with tag [tag]'); 134 | 135 | $provider->getRandomServerWithTag('tag'); 136 | } 137 | 138 | public function testClustersWithServersWithTag() 139 | { 140 | $serverOptionsWithTag = (new ServerOptions())->addTag('tag'); 141 | 142 | $serverWithTag = new Server('127.0.0.1', 8123, 'default', 'default', '', $serverOptionsWithTag); 143 | $serverWithoutTag = new Server('127.0.0.2'); 144 | 145 | $servers = [ 146 | $serverWithTag, 147 | $serverWithoutTag, 148 | ]; 149 | 150 | $cluster = new Cluster('test', $servers); 151 | 152 | $provider = new ServerProvider(); 153 | $provider->addCluster($cluster); 154 | 155 | $this->assertEquals($serverWithTag, $provider->getRandomServerFromClusterByTag('test', 'tag'), 'Correctly returns server from cluster by tag'); 156 | } 157 | 158 | public function testServerTagNotFoundInCluster() 159 | { 160 | $servers = [ 161 | new Server('127.0.0.1'), 162 | ]; 163 | $cluster = new Cluster('test', $servers); 164 | 165 | $provider = new ServerProvider(); 166 | $provider->addCluster($cluster); 167 | 168 | $this->expectException(ClusterException::class); 169 | $this->expectExceptionMessage('There are no servers with tag [tag] in cluster'); 170 | 171 | $provider->getRandomServerFromClusterByTag('test', 'tag'); 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /tests/ServerTest.php: -------------------------------------------------------------------------------- 1 | assertEquals('8123', $server->getPort(), 'Sets correct default HTTP port'); 19 | $this->assertEquals('default', $server->getDatabase(), 'Sets correct default database'); 20 | $this->assertEquals(null, $server->getUsername(), 'Sets correct default username'); 21 | $this->assertEquals(null, $server->getPassword(), 'Sets correct default password'); 22 | } 23 | 24 | public function testGetters() 25 | { 26 | $options = (new ServerOptions())->setProtocol('https'); 27 | $server = new Server('127.0.0.1', 8123, 'database', 'user', 'password', $options); 28 | 29 | $this->assertEquals('127.0.0.1', $server->getHost(), 'Sets correct host'); 30 | $this->assertEquals('8123', $server->getPort(), 'Sets correct port'); 31 | $this->assertEquals('database', $server->getDatabase(), 'Sets correct database'); 32 | $this->assertEquals('user', $server->getUsername(), 'Sets correct username'); 33 | $this->assertEquals('password', $server->getPassword(), 'Sets correct password'); 34 | $this->assertEquals($options, $server->getOptions(), 'Sets correct options'); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/TempTableTest.php: -------------------------------------------------------------------------------- 1 | 'UInt64', 'string' => 'String'], Format::TSV); 19 | 20 | $this->assertEquals('name', $file->getName()); 21 | $this->assertEquals(['number' => 'UInt64', 'string' => 'String'], $file->getStructure()); 22 | $this->assertEquals(Format::TSV, $file->getFormat()); 23 | } 24 | 25 | public function testFile() 26 | { 27 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 28 | $fileContent = []; 29 | $result = []; 30 | 31 | for ($i = 0; $i < 100; $i++) { 32 | $fileContent[] = $i."\tsome".PHP_EOL; 33 | 34 | $result[] = $i."\tsome".PHP_EOL; 35 | } 36 | 37 | $result = implode('', $result); 38 | 39 | file_put_contents($fileName, implode('', $fileContent)); 40 | 41 | $file = new TempTable('name', $fileName, ['number' => 'UInt64', 'string' => 'String'], Format::TSV); 42 | $stream = $file->open(false); 43 | 44 | $this->assertEquals($result, $stream->getContents(), 'Correctly reads content from file without encoding'); 45 | 46 | unlink($fileName); 47 | } 48 | 49 | public function testFileWithGzip() 50 | { 51 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 52 | $fileContent = []; 53 | $result = []; 54 | 55 | for ($i = 0; $i < 100; $i++) { 56 | $fileContent[] = $i."\tsome".PHP_EOL; 57 | 58 | $result[] = $i."\tsome".PHP_EOL; 59 | } 60 | 61 | $result = implode('', $result); 62 | 63 | file_put_contents($fileName, implode('', $fileContent)); 64 | 65 | $file = new TempTable('name', $fileName, ['number' => 'UInt64', 'string' => 'String'], Format::TSV); 66 | $stream = $file->open(); 67 | 68 | $this->assertEquals($result, gzdecode($stream->getContents()), 'Correctly reads content from file with encoding'); 69 | 70 | unlink($fileName); 71 | } 72 | 73 | public function testWithFileInterfaceAsSource() 74 | { 75 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 76 | $fileContent = []; 77 | $result = []; 78 | 79 | for ($i = 0; $i < 100; $i++) { 80 | $fileContent[] = $i.PHP_EOL; 81 | 82 | $result[] = $i.PHP_EOL; 83 | } 84 | 85 | $result = implode('', $result); 86 | 87 | file_put_contents($fileName, implode('', $fileContent)); 88 | 89 | $file = new File($fileName); 90 | $table = new TempTable('name', $file, ['number' => 'UInt64'], Format::TSV); 91 | 92 | $stream = $table->open(false); 93 | 94 | $this->assertEquals($result, $stream->getContents(), 'Correctly reads content from file without encoding'); 95 | 96 | unlink($fileName); 97 | } 98 | 99 | public function testWithFileInterfaceAsSourceWithGzip() 100 | { 101 | $fileName = tempnam(sys_get_temp_dir(), 'tbchc_'); 102 | $fileContent = []; 103 | $result = []; 104 | 105 | for ($i = 0; $i < 100; $i++) { 106 | $fileContent[] = $i.PHP_EOL; 107 | 108 | $result[] = $i.PHP_EOL; 109 | } 110 | 111 | $result = implode('', $result); 112 | 113 | file_put_contents($fileName, implode('', $fileContent)); 114 | 115 | $file = new File($fileName); 116 | $table = new TempTable('name', $file, ['number' => 'UInt64'], Format::TSV); 117 | 118 | $stream = $table->open(); 119 | 120 | $this->assertEquals($result, gzdecode($stream->getContents()), 'Correctly reads content from file with encoding'); 121 | 122 | unlink($fileName); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /utils/ccat/Makefile: -------------------------------------------------------------------------------- 1 | CC=g++ 2 | 3 | INSTALLBINDIR := ../../bin 4 | OS_NAME := $(shell uname -s | tr A-Z a-z) 5 | EXECUTABLE := "ccat_$(OS_NAME)" 6 | 7 | all: ccat 8 | 9 | ccat: ccat.cpp 10 | @echo "Compiling $<..."; $(CC) ccat.cpp -o $(EXECUTABLE) 11 | 12 | clean: 13 | rm -rf ../../bin/ccat 14 | 15 | install: 16 | @echo "Installing $(EXECUTABLE)..."; cp $(EXECUTABLE) $(INSTALLBINDIR) 17 | -------------------------------------------------------------------------------- /utils/ccat/ccat.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | # define ZERO_BYTE_TRANSFER_ERRNO 0 11 | 12 | #ifdef EINTR 13 | # define IS_EINTR(x) ((x) == EINTR) 14 | #else 15 | # define IS_EINTR(x) 0 16 | #endif 17 | 18 | enum { SYS_BUFSIZE_MAX = INT_MAX >> 20 << 20 }; 19 | 20 | size_t safe_write (int fd, void const *buf, size_t count) 21 | { 22 | for (;;) 23 | { 24 | ssize_t result = write (fd, buf, count); 25 | 26 | if (0 <= result) 27 | return result; 28 | else if (IS_EINTR (errno)) 29 | continue; 30 | else if (errno == EINVAL && SYS_BUFSIZE_MAX < count) 31 | count = SYS_BUFSIZE_MAX; 32 | else 33 | return result; 34 | } 35 | } 36 | 37 | size_t writeToStdOut (const void *buf, size_t count) 38 | { 39 | size_t total = 0; 40 | const char *ptr = (const char *) buf; 41 | 42 | while (count > 0) 43 | { 44 | size_t n_rw = safe_write (1, ptr, count); 45 | if (n_rw == (size_t) -1) 46 | break; 47 | if (n_rw == 0) 48 | { 49 | errno = ZERO_BYTE_TRANSFER_ERRNO; 50 | break; 51 | } 52 | total += n_rw; 53 | ptr += n_rw; 54 | count -= n_rw; 55 | } 56 | 57 | return total; 58 | } 59 | 60 | 61 | int main(int argc, char const *argv[]) 62 | { 63 | if (argc < 2) { 64 | std::cerr 65 | << "You must pass only one argument to the programm contained path to the file with files to read." 66 | << std::endl 67 | << "Example of usage: ccat file.txt" 68 | << std::endl; 69 | 70 | return EXIT_FAILURE; 71 | } 72 | 73 | //open file and get descriptor number 74 | FILE * file = fopen(argv[1], "r"); 75 | 76 | if (!file) { 77 | std::cerr << "failed to open " << argv[1] << std::endl; 78 | 79 | return EXIT_FAILURE; 80 | } 81 | 82 | int fileDescriptor = fileno(file); 83 | 84 | //Notify the system that we will sequentially read the file on that file descriptor 85 | #ifdef POSIX_FADV_SEQUENTIAL 86 | posix_fadvise(fileDescriptor, 0, 0, POSIX_FADV_SEQUENTIAL); 87 | #endif 88 | 89 | char * fileName = NULL; 90 | size_t len = 0; 91 | 92 | char *inbuf; 93 | 94 | size_t pageSize = getpagesize(); 95 | size_t insize = 128*1024; 96 | size_t n_read; 97 | 98 | /* 99 | Main loop with fast buffered read of files though syscals and output to stdout. 100 | */ 101 | while(getline(&fileName, &len, file) != -1) { 102 | 103 | //remove \n character for opening file. 104 | fileName[strcspn(fileName, "\n")] = 0; 105 | 106 | int fileWithDataDescriptor = open(fileName, O_RDONLY); 107 | 108 | if (fileWithDataDescriptor == -1) { 109 | 110 | perror("fopen"); 111 | std::cerr << fileName << std::endl; 112 | close(fileDescriptor); 113 | 114 | return EXIT_FAILURE; 115 | } 116 | 117 | #ifdef POSIX_FADV_SEQUENTIAL 118 | //Notify the system that we will sequentially read the file on that file descriptor 119 | posix_fadvise(fileWithDataDescriptor, 0, 0, POSIX_FADV_SEQUENTIAL); 120 | #endif 121 | 122 | inbuf = (char*) malloc(insize + pageSize - 1); 123 | 124 | while(n_read = read(fileWithDataDescriptor, inbuf, insize)) 125 | { 126 | writeToStdOut(inbuf, n_read); 127 | } 128 | 129 | free(inbuf); 130 | close(fileWithDataDescriptor); 131 | } 132 | 133 | close(fileDescriptor); 134 | 135 | return EXIT_SUCCESS; 136 | } 137 | 138 | --------------------------------------------------------------------------------