├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── administration.go ├── cmd └── esbulk │ └── main.go ├── doc.go ├── docs ├── 402504.cast ├── asciicast.gif ├── esbulk.1 ├── esbulk.md └── repo │ ├── authors.json │ ├── authors.png │ ├── cohorts.json │ ├── cohorts.png │ ├── survival.json │ └── survival.png ├── extra ├── Dockerfile ├── broken.jsonl └── ok.jsonl ├── fixtures ├── gen.py └── v10k.jsonl ├── flags.go ├── go.mod ├── go.sum ├── indexing.go ├── measurements.csv ├── packaging ├── debian │ └── esbulk │ │ └── DEBIAN │ │ └── control └── rpm │ ├── buildrpm.sh │ └── esbulk.spec ├── run.go └── run_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | 25 | coverage.out 26 | 27 | /esbulk 28 | packaging/debian/*deb 29 | packaging/debian/esbulk/usr/ 30 | esbulk-*rpm 31 | esbulk_*deb 32 | 33 | .vagrant/* 34 | Vagrantfile 35 | TODO.md 36 | /tmp 37 | /logs 38 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - tip 5 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We welcome any contribution, be it bug, maintenance or typo fixes, documentation updates, or bigger changes. All contributions are reviewed and discussed, if necessary. Thank you. 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TARGETS := esbulk 2 | VERSION := 0.7.18 3 | GOLDFLAGS := "-w -s" 4 | 5 | # testing against elasticsearch may require larger amounts of memory 6 | test: 7 | go test -cover -v 8 | 9 | imports: 10 | goimports -w . 11 | 12 | fmt: 13 | go fmt ./... 14 | 15 | all: fmt test 16 | go build -ldflags=$(GOLDFLAGS) -o esbulk cmd/esbulk/main.go 17 | 18 | install: 19 | go install 20 | 21 | clean: 22 | go clean 23 | rm -f coverage.out 24 | rm -f $(TARGETS) 25 | rm -f esbulk-*.x86_64.rpm 26 | rm -f packaging/debian/esbulk_*.deb 27 | rm -f esbulk_*.deb 28 | rm -rf packaging/debian/esbulk/usr 29 | rm -rf logs/ 30 | 31 | cover: 32 | go get -d && go test -v -coverprofile=coverage.out 33 | go tool cover -html=coverage.out 34 | 35 | esbulk: 36 | CGO_ENABLED=0 go build -ldflags=$(GOLDFLAGS) -o esbulk cmd/esbulk/main.go 37 | 38 | # ==== packaging 39 | 40 | image: 41 | DOCKER_CONTENT_TRUST=0 docker build --rm -t tirtir/esbulk:latest -t tirtir/esbulk:$(VERSION) . 42 | docker image prune --force --filter label=stage=intermediate 43 | 44 | rmi: 45 | docker rmi tirtir/esbulk:$(VERSION) 46 | 47 | deb: $(TARGETS) 48 | mkdir -p packaging/debian/esbulk/usr/sbin 49 | cp $(TARGETS) packaging/debian/esbulk/usr/sbin 50 | mkdir -p packaging/debian/esbulk/usr/local/share/man/man1 51 | cp docs/esbulk.1 packaging/debian/esbulk/usr/local/share/man/man1 52 | cd packaging/debian && fakeroot dpkg-deb --build esbulk . 53 | mv packaging/debian/esbulk*deb . 54 | 55 | rpm: $(TARGETS) 56 | mkdir -p $(HOME)/rpmbuild/{BUILD,SOURCES,SPECS,RPMS} 57 | cp ./packaging/rpm/esbulk.spec $(HOME)/rpmbuild/SPECS 58 | cp $(TARGETS) $(HOME)/rpmbuild/BUILD 59 | # md2man-roff docs/esbulk.md > docs/esbulk.1 60 | cp docs/esbulk.1 $(HOME)/rpmbuild/BUILD 61 | ./packaging/rpm/buildrpm.sh esbulk 62 | cp $(HOME)/rpmbuild/RPMS/x86_64/esbulk*.rpm . 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | esbulk 2 | ====== 3 | 4 | Fast parallel command line [bulk loading](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html) utility for elasticsearch. Data is read from a 5 | [newline delimited JSON](http://jsonlines.org/) file or stdin and indexed into elasticsearch in bulk 6 | *and* in parallel. The shortest command would be: 7 | 8 | ```shell 9 | $ esbulk -index my-index-name < file.ldj 10 | ``` 11 | 12 | Caveat: If indexing *pressure* on the bulk API is too high (dozens or hundreds of 13 | parallel workers, large batch sizes, depending on you setup), esbulk will halt 14 | and report an error: 15 | 16 | ```shell 17 | $ esbulk -index my-index-name -w 100 file.ldj 18 | 2017/01/02 16:25:25 error during bulk operation, try less workers (lower -w value) or 19 | increase thread_pool.bulk.queue_size in your nodes 20 | ``` 21 | 22 | Please note that, in such a case, some documents are indexed and some are not. 23 | Your index will be in an inconsistent state, since there is no transactional 24 | bracket around the indexing process. 25 | 26 | However, using defaults (parallelism: number of cores) on a single node setup 27 | will just work. For larger clusters, increase the number of workers until you 28 | see full CPU utilization. After that, more workers won't buy any more speed. 29 | 30 | Currently, esbulk is [tested against](https://git.io/Jzg2u) elasticsearch 31 | versions 2, 5, 6, 7 and 8 using 32 | [testcontainers](https://github.com/testcontainers/testcontainers-go). Originally written for [Leipzig University 33 | Library](https://en.wikipedia.org/wiki/Leipzig_University_Library), [project 34 | finc](https://finc.info). 35 | 36 | [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active) 37 | ![GitHub All Releases](https://img.shields.io/github/downloads/miku/esbulk/total.svg) 38 | 39 | Installation 40 | ------------ 41 | 42 | $ go install github.com/miku/esbulk/cmd/esbulk@latest 43 | 44 | For `deb` or `rpm` packages, see: https://github.com/miku/esbulk/releases 45 | 46 | Usage 47 | ----- 48 | 49 | $ esbulk -h 50 | Usage of esbulk: 51 | -0 set the number of replicas to 0 during indexing 52 | -c string 53 | create index mappings, settings, aliases, https://is.gd/3zszeu 54 | -cpuprofile string 55 | write cpu profile to file 56 | -id string 57 | name of field to use as id field, by default ids are autogenerated 58 | -index string 59 | index name 60 | -mapping string 61 | mapping string or filename to apply before indexing 62 | -memprofile string 63 | write heap profile to file 64 | -optype string 65 | optype (index - will replace existing data, 66 | create - will only create a new doc, 67 | update - create new or update existing data) 68 | (default "index") 69 | -p string 70 | pipeline to use to preprocess documents 71 | -purge 72 | purge any existing index before indexing 73 | -purge-pause duration 74 | pause after purge (default 1s) 75 | -r string 76 | Refresh interval after import (default "1s") 77 | -server value 78 | elasticsearch server, this works with https as well 79 | -size int 80 | bulk batch size (default 1000) 81 | -skipbroken 82 | skip broken json 83 | -type string 84 | elasticsearch doc type (deprecated since ES7) 85 | -u string 86 | http basic auth username:password, like curl -u 87 | -v prints current program version 88 | -verbose 89 | output basic progress 90 | -w int 91 | number of workers to use (default 8) 92 | -z unzip gz'd file on the fly 93 | 94 | ![](https://raw.githubusercontent.com/miku/esbulk/master/docs/asciicast.gif) 95 | 96 | To index a JSON file, that contains one document 97 | per line, just run: 98 | 99 | $ esbulk -index example file.ldj 100 | 101 | Where `file.ldj` is line delimited JSON, like: 102 | 103 | {"name": "esbulk", "version": "0.2.4"} 104 | {"name": "estab", "version": "0.1.3"} 105 | ... 106 | 107 | By default `esbulk` will use as many parallel 108 | workers, as there are cores. To tweak the indexing 109 | process, adjust the `-size` and `-w` parameters. 110 | 111 | You can index from gzipped files as well, using 112 | the `-z` flag: 113 | 114 | $ esbulk -z -index example file.ldj.gz 115 | 116 | Starting with 0.3.7 the preferred method to set a 117 | non-default server hostport is via `-server`, e.g. 118 | 119 | $ esbulk -server https://0.0.0.0:9201 120 | 121 | This way, you can use https as well, which was not 122 | possible before. Options `-host` and `-port` are 123 | gone as of [esbulk 0.5.0](https://github.com/miku/esbulk/releases/tag/v0.5.0). 124 | 125 | Reusing IDs 126 | ----------- 127 | 128 | Since version 0.3.8: If you want to reuse IDs from your documents in elasticsearch, you 129 | can specify the ID field via `-id` flag: 130 | 131 | $ cat file.json 132 | {"x": "doc-1", "db": "mysql"} 133 | {"x": "doc-2", "db": "mongo"} 134 | 135 | Here, we would like to reuse the ID from field *x*. 136 | 137 | $ esbulk -id x -index throwaway -verbose file.json 138 | ... 139 | 140 | $ curl -s http://localhost:9200/throwaway/_search | jq 141 | { 142 | "took": 2, 143 | "timed_out": false, 144 | "_shards": { 145 | "total": 5, 146 | "successful": 5, 147 | "failed": 0 148 | }, 149 | "hits": { 150 | "total": 2, 151 | "max_score": 1, 152 | "hits": [ 153 | { 154 | "_index": "throwaway", 155 | "_type": "default", 156 | "_id": "doc-2", 157 | "_score": 1, 158 | "_source": { 159 | "x": "doc-2", 160 | "db": "mongo" 161 | } 162 | }, 163 | { 164 | "_index": "throwaway", 165 | "_type": "default", 166 | "_id": "doc-1", 167 | "_score": 1, 168 | "_source": { 169 | "x": "doc-1", 170 | "db": "mysql" 171 | } 172 | } 173 | ] 174 | } 175 | } 176 | 177 | Nested ID fields 178 | ---------------- 179 | 180 | Version 0.4.3 adds support for nested ID fields: 181 | 182 | ``` 183 | $ cat fixtures/pr-8-1.json 184 | {"a": {"b": 1}} 185 | {"a": {"b": 2}} 186 | {"a": {"b": 3}} 187 | 188 | $ esbulk -index throwaway -id a.b < fixtures/pr-8-1.json 189 | ... 190 | ``` 191 | 192 | Concatenated ID 193 | --------------- 194 | 195 | Version 0.4.3 adds support for IDs that are the concatenation of multiple fields: 196 | 197 | ``` 198 | $ cat fixtures/pr-8-2.json 199 | {"a": {"b": 1}, "c": "a"} 200 | {"a": {"b": 2}, "c": "b"} 201 | {"a": {"b": 3}, "c": "c"} 202 | 203 | $ esbulk -index throwaway -id a.b,c < fixtures/pr-8-1.json 204 | ... 205 | 206 | { 207 | "_index": "xxx", 208 | "_type": "default", 209 | "_id": "1a", 210 | "_score": 1, 211 | "_source": { 212 | "a": { 213 | "b": 1 214 | }, 215 | "c": "a" 216 | } 217 | }, 218 | ``` 219 | 220 | Using X-Pack 221 | ------------ 222 | 223 | Since 0.4.2: support for secured elasticsearch nodes: 224 | 225 | ``` 226 | $ esbulk -u elastic:changeme -index myindex file.ldj 227 | ``` 228 | 229 | ---- 230 | 231 | A similar project has been started for solr, called [solrbulk](https://github.com/miku/solrbulk). 232 | 233 | Contributors 234 | ------------ 235 | 236 | * [klaubert](https://github.com/klaubert) 237 | * [sakshambathla](https://github.com/sakshambathla) 238 | * [mumoshu](https://github.com/mumoshu) 239 | * [albertpastrana](https://github.com/albertpastrana) 240 | * [faultlin3](https://github.com/faultlin3) 241 | * [gransy](https://github.com/gransy) 242 | * [Christoph Kepper](https://github.com/ckepper) 243 | * Christian Solomon 244 | * Mikael Byström 245 | 246 | and others. 247 | 248 | Measurements 249 | ------------ 250 | 251 | ```shell 252 | $ csvlook -I measurements.csv 253 | | es | esbulk | docs | avg_b | nodes | cores | total_heap_gb | t_s | docs_per_s | repl | 254 | |-------|--------|-----------|-------|-------|-------|---------------|-------|------------|------| 255 | | 6.1.2 | 0.4.8 | 138000000 | 2000 | 1 | 32 | 64 | 6420 | 22100 | 1 | 256 | | 6.1.2 | 0.4.8 | 138000000 | 2000 | 1 | 8 | 30 | 27360 | 5100 | 1 | 257 | | 6.1.2 | 0.4.8 | 1000000 | 2000 | 1 | 4 | 1 | 300 | 3300 | 1 | 258 | | 6.1.2 | 0.4.8 | 10000000 | 26 | 1 | 4 | 8 | 122 | 81000 | 1 | 259 | | 6.1.2 | 0.4.8 | 10000000 | 26 | 1 | 32 | 64 | 32 | 307000 | 1 | 260 | | 6.2.3 | 0.4.10 | 142944530 | 2000 | 2 | 64 | 128 | 26253 | 5444 | 1 | 261 | | 6.2.3 | 0.4.10 | 142944530 | 2000 | 2 | 64 | 128 | 11113 | 12831 | 0 | 262 | | 6.2.3 | 0.4.13 | 15000000 | 6000 | 2 | 64 | 128 | 2460 | 6400 | 0 | 263 | ``` 264 | 265 | Why not add a [row](https://github.com/miku/esbulk/pulls)? 266 | -------------------------------------------------------------------------------- /administration.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 by Leipzig University Library, http://ub.uni-leipzig.de 2 | // The Finc Authors, http://finc.info 3 | // Martin Czygan, 4 | // 5 | // This file is part of some open source application. 6 | // 7 | // Some open source application is free software: you can redistribute 8 | // it and/or modify it under the terms of the GNU General Public 9 | // License as published by the Free Software Foundation, either 10 | // version 3 of the License, or (at your option) any later version. 11 | // 12 | // Some open source application is distributed in the hope that it will 13 | // be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 14 | // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with Foobar. If not, see . 19 | // 20 | // @license GPL-3.0+ 21 | 22 | package esbulk 23 | 24 | import ( 25 | "fmt" 26 | "log" 27 | "net/http" 28 | 29 | "github.com/segmentio/encoding/json" 30 | "github.com/sethgrid/pester" 31 | ) 32 | 33 | // FlushIndex flushes index. 34 | func FlushIndex(idx int, options Options) error { 35 | server := options.Servers[idx] 36 | link := fmt.Sprintf("%s/%s/_flush", server, options.Index) 37 | req, err := http.NewRequest("POST", link, nil) 38 | if err != nil { 39 | return err 40 | } 41 | if options.Username != "" && options.Password != "" { 42 | req.SetBasicAuth(options.Username, options.Password) 43 | } 44 | req.Header.Set("Content-Type", "application/json") 45 | resp, err := pester.Do(req) 46 | if err != nil { 47 | return err 48 | } 49 | if options.Verbose { 50 | log.Printf("index flushed: %s\n", resp.Status) 51 | } 52 | return nil 53 | } 54 | 55 | // GetSettings fetches the settings of the index. 56 | func GetSettings(idx int, options Options) (map[string]interface{}, error) { 57 | server := options.Servers[idx] 58 | link := fmt.Sprintf("%s/%s/_settings", server, options.Index) 59 | 60 | req, err := http.NewRequest("GET", link, nil) 61 | if err != nil { 62 | return nil, err 63 | } 64 | if options.Username != "" && options.Password != "" { 65 | req.SetBasicAuth(options.Username, options.Password) 66 | } 67 | req.Header.Set("Content-Type", "application/json") 68 | resp, err := pester.Do(req) 69 | if err != nil { 70 | return nil, err 71 | } 72 | defer resp.Body.Close() 73 | if resp.StatusCode != 200 { 74 | return nil, fmt.Errorf("could not get settings: %s", link) 75 | } 76 | 77 | doc := make(map[string]interface{}) 78 | dec := json.NewDecoder(resp.Body) 79 | if err := dec.Decode(&doc); err != nil { 80 | return nil, fmt.Errorf("failed to decode settings: %v", err) 81 | } 82 | // Example response. 83 | // { 84 | // "ai": { 85 | // "settings": { 86 | // "index": { 87 | // "refresh_interval": "1s", 88 | // "number_of_shards": "5", 89 | // "provided_name": "ai", 90 | // "creation_date": "1523372145102", 91 | // "number_of_replicas": "1", 92 | // "uuid": "5k-id0OZTKKU4A7DeeUNdQ", 93 | // "version": { 94 | // "created": "6020399" 95 | // } 96 | // } 97 | // } 98 | // } 99 | // } 100 | 101 | return doc, nil 102 | } 103 | -------------------------------------------------------------------------------- /cmd/esbulk/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 by Leipzig University Library, http://ub.uni-leipzig.de 2 | // The Finc Authors, http://finc.info 3 | // Martin Czygan, 4 | // 5 | // This file is part of some open source application. 6 | // 7 | // Some open source application is free software: you can redistribute 8 | // it and/or modify it under the terms of the GNU General Public 9 | // License as published by the Free Software Foundation, either 10 | // version 3 of the License, or (at your option) any later version. 11 | // 12 | // Some open source application is distributed in the hope that it will 13 | // be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 14 | // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with Foobar. If not, see . 19 | // 20 | // @license GPL-3.0+ 21 | 22 | package main 23 | 24 | import ( 25 | "flag" 26 | "log" 27 | "os" 28 | "runtime" 29 | "strings" 30 | "time" 31 | 32 | "github.com/miku/esbulk" 33 | ) 34 | 35 | var ( 36 | version = flag.Bool("v", false, "prints current program version") 37 | cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file") 38 | memprofile = flag.String("memprofile", "", "write heap profile to file") 39 | indexName = flag.String("index", "", "index name") 40 | opType = flag.String("optype", "index", "optype (index - will replace existing data, create - will only create a new doc, update - create new or update existing data)") 41 | docType = flag.String("type", "", "elasticsearch doc type (deprecated since ES7)") 42 | batchSize = flag.Int("size", 1000, "bulk batch size") 43 | numWorkers = flag.Int("w", runtime.NumCPU(), "number of workers to use") 44 | verbose = flag.Bool("verbose", false, "output basic progress") 45 | skipbroken = flag.Bool("skipbroken", false, "skip broken json") 46 | gzipped = flag.Bool("z", false, "unzip gz'd file on the fly") 47 | mapping = flag.String("mapping", "", "mapping string or filename to apply before indexing") 48 | config = flag.String("c", "", "create index mappings, settings, aliases, https://is.gd/3zszeu") 49 | purge = flag.Bool("purge", false, "purge any existing index before indexing") 50 | purgePause = flag.Duration("purge-pause", 1*time.Second, "pause after purge") 51 | idfield = flag.String("id", "", "name of field to use as id field, by default ids are autogenerated") 52 | user = flag.String("u", "", "http basic auth username:password, like curl -u") 53 | zeroReplica = flag.Bool("0", false, "set the number of replicas to 0 during indexing") 54 | refreshInterval = flag.String("r", "1s", "Refresh interval after import") 55 | pipeline = flag.String("p", "", "pipeline to use to preprocess documents") 56 | serverFlags esbulk.ArrayFlags 57 | ) 58 | 59 | func main() { 60 | flag.Var(&serverFlags, "server", "elasticsearch server, this works with https as well") 61 | flag.Parse() 62 | var ( 63 | file *os.File = os.Stdin 64 | username, password string 65 | ) 66 | if flag.NArg() > 0 { 67 | f, err := os.Open(flag.Arg(0)) 68 | if err != nil { 69 | log.Fatalln(err) 70 | } 71 | defer f.Close() 72 | file = f 73 | } 74 | if len(*user) > 0 { 75 | parts := strings.Split(*user, ":") 76 | if len(parts) != 2 { 77 | log.Fatal("http basic auth syntax is: username:password") 78 | } 79 | username = parts[0] 80 | password = parts[1] 81 | } 82 | runner := &esbulk.Runner{ 83 | BatchSize: *batchSize, 84 | Config: *config, 85 | CpuProfile: *cpuprofile, 86 | DocType: *docType, 87 | File: file, 88 | FileGzipped: *gzipped, 89 | IdentifierField: *idfield, 90 | IndexName: *indexName, 91 | Mapping: *mapping, 92 | MemProfile: *memprofile, 93 | NumWorkers: *numWorkers, 94 | OpType: *opType, 95 | Password: password, 96 | Pipeline: *pipeline, 97 | Purge: *purge, 98 | PurgePause: *purgePause, 99 | RefreshInterval: *refreshInterval, 100 | Servers: serverFlags, 101 | ShowVersion: *version, 102 | SkipBroken: *skipbroken, 103 | Username: username, 104 | Verbose: *verbose, 105 | ZeroReplica: *zeroReplica, 106 | } 107 | if err := runner.Run(); err != nil { 108 | log.Fatal(err) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package esbulk implements a few helpers for performant indexing operations 2 | // for elasticsearch. 3 | package esbulk 4 | -------------------------------------------------------------------------------- /docs/402504.cast: -------------------------------------------------------------------------------- 1 | {"version": 2, "width": 161, "height": 30, "timestamp": 1616709235, "env": {"SHELL": "/bin/bash", "TERM": "xterm-256color"}} 2 | [1.191436, "o", "tir@trieste:~/code/miku/esbulk \u001b[38;5;130m[git:master] \u001b[m$ "] 3 | [2.015554, "o", "t"] 4 | [2.124784, "o", "m"] 5 | [2.323176, "o", "u"] 6 | [2.435671, "o", "x"] 7 | [2.524505, "o", " "] 8 | [2.835836, "o", "a"] 9 | [2.884155, "o", "t"] 10 | [3.010902, "o", "\r\n"] 11 | [3.022208, "o", "\u001b[?1h\u001b=\u001b[H\u001b[2J\u001b[?12l\u001b[?25h\u001b[?1000l\u001b[?1002l\u001b[?1006l\u001b[?1005l\u001b[?1004h\u001b[c\u001b(B\u001b[m\u001b[?12;25h\u001b[?12l\u001b[?25h\u001b[?1003l\u001b[?1006l\u001b[?2004l\u001b[1;1H\u001b[1;30r\u001b]112\u0007\u001b[1;3H"] 12 | [3.028452, "o", "\u001b[?25l\u001b[H$ \u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\u001b[1m\u001b[2m\r\n 0\u001b[38;5;250m:\u001b[38;5;255mbash\u001b[38;5;50m* \u001b(B\u001b[m\u001b[2m \u001b[38;5;233m\u001b[48;5;245m\u001b[1m <'/home/tir/.tmux/plugins/tmux-battery/scripts/battery_percentage.sh' not ready> \u001b[48;5;241m 25/03 \u001b[48;5;245m 22:53 \u001b(B\u001b[m\u001b[1;3H\u001b[?12l\u001b[?25h"] 13 | [3.029192, "o", "\u001b(B\u001b[m\u001b[?12;25h\u001b[?12l\u001b[?25h\u001b[?1003l\u001b[?1006l\u001b[?2004l\u001b[1;1H\u001b[1;30r\u001b[1;3H"] 14 | [3.03104, "o", "\u001b[?25l\u001b[H$ \u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\u001b[1m\u001b[2m\r\n 0\u001b[38;5;250m:\u001b[38;5;255mbash\u001b[38;5;50m* \u001b(B\u001b[m\u001b[2m \u001b[38;5;233m\u001b[48;5;245m\u001b[1m <'/home/tir/.tmux/plugins/tmux-battery/scripts/battery_percentage.sh' not ready> \u001b[48;5;241m 25/03 \u001b[48;5;245m 22:53 \u001b(B\u001b[m\u001b[1;3H\u001b[?12l\u001b[?25h"] 15 | [3.055307, "o", "\u001b[?25l\u001b[1m\u001b[2m\u001b[30;1H 0\u001b[38;5;250m:\u001b[38;5;255mbash\u001b[38;5;50m* \u001b(B\u001b[m\u001b[2m \u001b[38;5;233m\u001b[48;5;245m\u001b[1m 60% \u001b[48;5;241m 25/03 \u001b[48;5;245m 22:53 \u001b(B\u001b[m\u001b[1;3H\u001b[?12l\u001b[?25h"] 16 | [5.679862, "o", "p"] 17 | [5.746974, "o", "w"] 18 | [5.808603, "o", "d"] 19 | [5.933228, "o", "\r\n"] 20 | [5.936384, "o", "/home/tir/code/miku/esbulk\r\n"] 21 | [5.963902, "o", "$ "] 22 | [6.622515, "o", "\u001b[1;29r\u001b[2;29r\u001b[28S\u001b[1;1H\u001b[K$ \u001b[1;30r\u001b[1;3H"] 23 | [7.126049, "o", "m"] 24 | [7.255798, "o", "a"] 25 | [7.621056, "o", "k"] 26 | [7.795501, "o", "e"] 27 | [7.886829, "o", " "] 28 | [8.026092, "o", "t"] 29 | [8.112072, "o", "e"] 30 | [8.291392, "o", "s"] 31 | [8.338761, "o", "t"] 32 | [8.90118, "o", "\r\n"] 33 | [8.910917, "o", "go get -d && go test -v\r\n"] 34 | [9.628196, "o", "\u001b[?25l\u001b[1m\u001b[2m\u001b[30d 0\u001b[38;5;250m:\u001b[38;5;255mmake\u001b[38;5;50m* \u001b(B\u001b[m\u001b[2m \u001b[38;5;233m\u001b[48;5;245m\u001b[1m 60% \u001b[48;5;241m 25/03 \u001b[48;5;245m 22:54 \u001b(B\u001b[m\u001b[3;1H\u001b[?12l\u001b[?25h"] 35 | [10.549342, "o", "=== RUN TestIncompleteConfig\r\n"] 36 | [12.752496, "o", "--- PASS: TestIncompleteConfig (2.20s)\r\n=== RUN TestMinimalConfig\r\n"] 37 | [12.832779, "o", "2021/03/25 22:54:08 Starting container id: 58a040bbcfe5 image: quay.io/testcontainers/ryuk:0.2.3\r\n"] 38 | [13.612885, "o", "2021/03/25 22:54:09 Waiting for container id 58a040bbcfe5 image: quay.io/testcontainers/ryuk:0.2.3\r\n"] 39 | [13.919337, "o", "2021/03/25 22:54:09 Container is ready id: 58a040bbcfe5 image: quay.io/testcontainers/ryuk:0.2.3\r\n"] 40 | [13.969244, "o", "2021/03/25 22:54:09 Starting container id: 7b9fa118ed0a image: elasticsearch:7.11.2\r\n"] 41 | [14.774665, "o", "2021/03/25 22:54:10 Waiting for container id 7b9fa118ed0a image: elasticsearch:7.11.2\r\n"] 42 | [30.201141, "o", "2021/03/25 22:54:25 Container is ready id: 7b9fa118ed0a image: elasticsearch:7.11.2\r\n"] 43 | [30.307415, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[11d run_test.go:94: {\r\n \"name\" : \"7b9fa118ed0a\",\r\n \"cluster_name\" : \"docker-cluster\",\r\n \"cluster_uuid\" : \"Huqn3nJvSvih0TyBvTJsQg\",\r\n \"version\" : {\r\n \"number\" : \"7.11.2\",\r\n \"build_flavor\" : \"default\",\r\n \"build_type\" : \"docker\",\r\n \"build_hash\" : \"3e5a16cfec50876d20ea77b075070932c6464c7d\",\r\n \"build_date\" : \"2021-03-06T05:54:38.141101Z\",\r\n \"build_snapshot\" : false,\r\n \"lucene_version\" : \"8.7.0\",\r\n \"minimum_wire_compatibility_version\" : \"6.8.0\",\r\n \"minimum_index_compatibility_version\" : \"6.0.0-beta1\"\r\n },\r\n \"tagline\" : \"You Know, for Search\"\r\n }\r\n run_test.go:133: server should be up at http://localhost:39200\u001b[1;30r\u001b[29;1H"] 44 | [30.307603, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:26 using 1 server(s)\r\n2021/03/25 22:54:26 {[http://localhost:39200] abc any 5000 true http }\u001b[1;30r\u001b[29;1H"] 45 | [31.821018, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:27 created index: 200 OK\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:27 started 1 workers\u001b[1;30r\u001b[29;1H"] 46 | [31.827593, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:27 on shutdown, number_of_replicas will be set back to 1\r\n2021/03/25 22:54:27 on shutdown, refresh_interval will be set back to 1s\u001b[1;30r\u001b[29;1H"] 47 | [31.918354, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:27 applied setting: {\"index\": {\"refresh_interval\": \"-1\"}} with status 200 OK\r\n2021/03/25 22:54:27 start reading from fixtures/v10k.jsonl\u001b[1;30r\u001b[29;1H"] 48 | [31.92339, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:27 message content-length will be 293890\u001b[1;30r\u001b[29;1H"] 49 | [32.587714, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:28 [worker-0] @5000\u001b[1;30r\u001b[29;1H"] 50 | [32.592852, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:28 message content-length will be 295000\u001b[1;30r\u001b[29;1H"] 51 | [32.937253, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:28 [worker-0] @10000\r\n2021/03/25 22:54:28 10000 docs in 1.02s at 9807.954 docs/s with 1 workers\u001b[1;30r\u001b[29;1H"] 52 | [32.981304, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:28 applied setting: {\"index\": {\"refresh_interval\": \"1s\"}} with status 200 OK\u001b[1;30r\u001b[29;1H"] 53 | [32.988089, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:28 applied setting: {\"index\": {\"number_of_replicas\": \"1\"}} with status 200 OK\u001b[1;30r\u001b[29;1H"] 54 | [33.188247, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:28 index flushed: 200 OK\u001b[1;30r\u001b[29;1H"] 55 | [33.338721, "o", "\u001b[1;29r\u001b[7S\u001b[22d run_test.go:94: {\"took\":138,\"timed_out\":false,\"_shards\":{\"total\":1,\"successful\":1,\"skipped\":0,\"failed\":0},\"hits\":{\"total\":{\"value\":10000,\"relation\":\"eq\"},\"max_score\":1.0,\"hits\":[{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"x7Jha3gBPNkI9HmWsRUc\",\"_score\":1.0,\"_source\":{\"v\": \"0\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"yLJha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_source\":{\"v\": \"1\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"ybJha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_source\":{\"v\": \"2\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"yrJha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_source\":{\"v\": \"3\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"y7Jha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_source\":{\"v\": \"4\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"zLJha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_source\":{\"v\": \"5\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"zbJha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_source\":{\"v\": \"6\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"zrJha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_source\":{\"v\": \"7\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"z7Jha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_"] 56 | [33.338792, "o", "source\":{\"v\": \"8\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"0LJha3gBPNkI9HmWsRUd\",\"_score\":1.0,\"_source\":{\"v\": \"9\"}}]}}\u001b[1;30r\u001b[29;1H"] 57 | [33.341175, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:29 logging to logs/20210325225429-esbulk-test-es-7.11.2.log\u001b[1;30r\u001b[29;1H"] 58 | [34.084645, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:29 Starting container id: 7e88854f14d9 image: elasticsearch:6.8.14\u001b[1;30r\u001b[29;1H"] 59 | [34.735789, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:30 Waiting for container id 7e88854f14d9 image: elasticsearch:6.8.14\u001b[1;30r\u001b[29;1H"] 60 | [43.931274, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:39 Container is ready id: 7e88854f14d9 image: elasticsearch:6.8.14\u001b[1;30r\u001b[29;1H"] 61 | [44.04469, "o", "\u001b[1;29r\u001b[17S\u001b[12d run_test.go:94: {\r\n \"name\" : \"Phs8q_K\",\r\n \"cluster_name\" : \"docker-cluster\",\r\n \"cluster_uuid\" : \"zBfBOzSsSrG9GewFO9SD7Q\",\r\n \"version\" : {\r\n \"number\" : \"6.8.14\",\r\n \"build_flavor\" : \"default\",\r\n \"build_type\" : \"docker\",\r\n \"build_hash\" : \"dab5822\",\r\n \"build_date\" : \"2021-02-02T19:58:04.182039Z\",\r\n \"build_snapshot\" : false,\r\n \"lucene_version\" : \"7.7.3\",\r\n \"minimum_wire_compatibility_version\" : \"5.6.0\",\r\n \"minimum_index_compatibility_version\" : \"5.0.0\"\r\n },\r\n \"tagline\" : \"You Know, for Search\"\r\n }\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[3S\u001b[26d run_test.go:133: server should be up at http://localhost:39200\r\n2021/03/25 22:54:39 using 1 server(s)\r\n2021/03/25 22:54:39 {[http://localhost:39200] abc any 5000 true http }\u001b[1;30r\u001b[29;1H"] 62 | [44.833286, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:40 created index: 200 OK\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:40 started 1 workers\u001b[1;30r\u001b[29;1H"] 63 | [44.854714, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:40 on shutdown, number_of_replicas will be set back to 1\r\n2021/03/25 22:54:40 on shutdown, refresh_interval will be set back to 1s\u001b[1;30r\u001b[29;1H"] 64 | [44.892138, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:40 applied setting: {\"index\": {\"refresh_interval\": \"-1\"}} with status 200 OK\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:40 start reading from fixtures/v10k.jsonl\u001b[1;30r\u001b[29;1H"] 65 | [44.898929, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:40 message content-length will be 293890\u001b[1;30r\u001b[29;1H"] 66 | [45.745394, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:41 [worker-0] @5000\u001b[1;30r\u001b[29;1H"] 67 | [45.750598, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:41 message content-length will be 295000\u001b[1;30r\u001b[29;1H"] 68 | [45.960621, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:41 [worker-0] @10000\r\n2021/03/25 22:54:41 10000 docs in 1.07s at 9359.287 docs/s with 1 workers\u001b[1;30r\u001b[29;1H"] 69 | [45.981572, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:41 applied setting: {\"index\": {\"refresh_interval\": \"1s\"}} with status 200 OK\u001b[1;30r\u001b[29;1H"] 70 | [46.013876, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:41 applied setting: {\"index\": {\"number_of_replicas\": \"1\"}} with status 200 OK\u001b[1;30r\u001b[29;1H"] 71 | [46.346113, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:42 index flushed: 200 OK\u001b[1;30r\u001b[29;1H"] 72 | [46.421336, "o", "\u001b[1;29r\u001b[7S\u001b[22d run_test.go:94: {\"took\":58,\"timed_out\":false,\"_shards\":{\"total\":5,\"successful\":5,\"skipped\":0,\"failed\":0},\"hits\":{\"total\":10000,\"max_score\":1.0,\"hits\":[{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"DA9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"0\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"Ew9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"7\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"Fg9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"10\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"GA9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"12\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"Gg9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"14\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"HA9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"16\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"IA9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"20\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"IQ9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"21\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"Kg9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"30\"}},"] 73 | [46.421546, "o", "{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"Mg9ha3gBD232FCvj43fc\",\"_score\":1.0,\"_source\":{\"v\": \"38\"}}]}}\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[29;1H\n\u001b[A run_test.go:178: es6 detected\u001b[1;30r\u001b[29;1H"] 74 | [46.422791, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:42 logging to logs/20210325225442-esbulk-test-es-6.8.14.log\u001b[1;30r\u001b[29;1H"] 75 | [46.92474, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:42 Starting container id: f2d6f5604d1a image: elasticsearch:5.6.16\u001b[1;30r\u001b[29;1H"] 76 | [47.637161, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:43 Waiting for container id f2d6f5604d1a image: elasticsearch:5.6.16\u001b[1;30r\u001b[29;1H"] 77 | [55.624341, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:51 Container is ready id: f2d6f5604d1a image: elasticsearch:5.6.16\u001b[1;30r\u001b[29;1H"] 78 | [55.756573, "o", "\u001b[1;29r\u001b[14S\u001b[15d run_test.go:94: {\r\n \"name\" : \"CizMOsQ\",\r\n \"cluster_name\" : \"elasticsearch\",\r\n \"cluster_uuid\" : \"ZH68DTPNQvCPRjMxsraP6A\",\r\n \"version\" : {\r\n \"number\" : \"5.6.16\",\r\n \"build_hash\" : \"3a740d1\",\r\n \"build_date\" : \"2019-03-13T15:33:36.565Z\",\r\n \"build_snapshot\" : false,\r\n \"lucene_version\" : \"6.6.1\"\r\n },\r\n \"tagline\" : \"You Know, for Search\"\r\n }\r\n run_test.go:133: server should be up at http://localhost:39200\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:51 using 1 server(s)\r\n2021/03/25 22:54:51 {[http://localhost:39200] abc any 5000 true http }\u001b[1;30r\u001b[29;1H"] 79 | [56.13315, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:51 created index: 200 OK\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:51 started 1 workers\u001b[1;30r\u001b[29;1H"] 80 | [56.146621, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:51 on shutdown, number_of_replicas will be set back to 1\r\n2021/03/25 22:54:51 on shutdown, refresh_interval will be set back to 1s\u001b[1;30r\u001b[29;1H"] 81 | [56.185491, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:51 applied setting: {\"index\": {\"refresh_interval\": \"-1\"}} with status 200 OK\r\n2021/03/25 22:54:51 start reading from fixtures/v10k.jsonl\u001b[1;30r\u001b[29;1H"] 82 | [56.192082, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:51 message content-length will be 293890\u001b[1;30r\u001b[29;1H"] 83 | [56.919142, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:52 [worker-0] @5000\u001b[1;30r\u001b[29;1H"] 84 | [56.924784, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:52 message content-length will be 295000\u001b[1;30r\u001b[29;1H"] 85 | [57.151738, "o", "\u001b[1;29r\u001b[2S\u001b[27d2021/03/25 22:54:52 [worker-0] @10000\r\n2021/03/25 22:54:52 10000 docs in 0.97s at 10350.515 docs/s with 1 workers\u001b[1;30r\u001b[29;1H"] 86 | [57.168951, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:52 applied setting: {\"index\": {\"refresh_interval\": \"1s\"}} with status 200 OK\u001b[1;30r\u001b[29;1H"] 87 | [57.199809, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:52 applied setting: {\"index\": {\"number_of_replicas\": \"1\"}} with status 200 OK\u001b[1;30r\u001b[29;1H"] 88 | [57.53695, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:53 index flushed: 200 OK\u001b[1;30r\u001b[29;1H"] 89 | [57.603163, "o", "\u001b[1;29r\u001b[7S\u001b[22d run_test.go:94: {\"took\":49,\"timed_out\":false,\"_shards\":{\"total\":5,\"successful\":5,\"skipped\":0,\"failed\":0},\"hits\":{\"total\":10000,\"max_score\":1.0,\"hits\":[{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma3u\",\"_score\":1.0,\"_source\":{\"v\": \"2\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma3v\",\"_score\":1.0,\"_source\":{\"v\": \"3\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma35\",\"_score\":1.0,\"_source\":{\"v\": \"13\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma38\",\"_score\":1.0,\"_source\":{\"v\": \"16\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4C\",\"_score\":1.0,\"_source\":{\"v\": \"22\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4E\",\"_score\":1.0,\"_source\":{\"v\": \"24\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4T\",\"_score\":1.0,\"_source\":{\"v\": \"39\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4X\",\"_score\":1.0,\"_source\":{\"v\": \"43\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4a\",\"_score\":1.0,\"_source\":{\"v\": \"46\"}},"] 90 | [57.6034, "o", "{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4d\",\"_score\":1.0,\"_source\":{\"v\": \"49\"}}]}}\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[29;1H\n\u001b[A run_test.go:178: es6 detected\u001b[1;30r\u001b[29;1H"] 91 | [57.605147, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[A2021/03/25 22:54:53 logging to logs/20210325225453-esbulk-test-es-5.6.16.log\u001b[1;30r\u001b[29;1H"] 92 | [58.150343, "o", "\u001b[1;29r\u001b[2S\u001b[27d--- PASS: TestMinimalConfig (45.40s)\r\nPASS\u001b[1;30r\u001b[29;1H"] 93 | [58.153541, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[Aok \u001b[4Cgithub.com/miku/esbulk\u001b[2C47.609s\u001b[1;30r\u001b[29;1H"] 94 | [58.21801, "o", "$ "] 95 | [58.692843, "o", "\u001b[?25l\u001b[1m\u001b[2m\r\n 0\u001b[38;5;250m:\u001b[38;5;255mbash\u001b[38;5;50m* \u001b(B\u001b[m\u001b[2m \u001b[38;5;233m\u001b[48;5;245m\u001b[1m 60% \u001b[48;5;241m 25/03 \u001b[48;5;245m 22:54 \u001b(B\u001b[m\u001b[29;3H\u001b[?12l\u001b[?25h"] 96 | [64.090416, "o", "l"] 97 | [64.197017, "o", "e"] 98 | [64.387099, "o", "s"] 99 | [64.521325, "o", "s"] 100 | [64.641873, "o", " "] 101 | [64.858378, "o", "logs/20210325225453-esbulk-test-es-5.6.16.log"] 102 | [65.203184, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[1;30r\u001b[29;1H"] 103 | [65.214854, "o", "\"logs/20210325225453-esbulk-test-es-5.6.16.log\" may be a binary file. See it anyway? "] 104 | [65.700653, "o", "\u001b[?25l\u001b[1m\u001b[2m\r\n 0\u001b[38;5;250m:\u001b[38;5;255mless\u001b[38;5;50m* \u001b(B\u001b[m\u001b[2m \u001b[38;5;233m\u001b[48;5;245m\u001b[1m 60% \u001b[48;5;241m 25/03 \u001b[48;5;245m 22:55 \u001b(B\u001b[m\u001b[29;87H\u001b[?12l\u001b[?25h"] 105 | [65.902216, "o", "\u001b[?25l\u001b[H\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\r\n\u001b[K\u001b[?12l\u001b[?25h\u001b[1;29r\u001b[8S\u001b[21d\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mP[2021-03-25T21:54:45,415][INFO ][o.e.n.Node ] [] initializing ...\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[m[2021-03-25T21:54:45,506][INFO ][o.e.e.NodeEnvironment ] [CizMOsQ] using [1] data paths, mounts [[/usr/share/elasticsearch/data (/dev/nvme0n \r1p2)]], net usable_space [794.7gb], net total_space [1.7tb], spins? [possibly], types [ext4]\r\n\u001b[7m^A^@^@^@^@^@^@<84>\u001b(B\u001b[m[2021-03-25T21:54:45,506][INFO ][o.e.e.NodeEnvironment ] [CizMOsQ] heap size [1.9gb], compressed ordinary object pointers [true]\r\n\u001b[7m^A^@^@^@^@^@^@<9B>\u001b(B\u001b[m[2021-03-25T21:54:45,507][INFO ][o.e.n.Node ] node name [CizMOsQ] derived from node ID [CizMOsQ0SkKB03BACJ6KWA]; set [node.name] \rto override\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[m[2021-03-25T21:54:45,508][INFO ][o.e.n.Node ] version[5.6.16], pid[1], build[3a740"] 106 | [65.902352, "o", "d1/2019-03-13T15:33:36.565Z], OS[Linux/5.4.0-70 \r-generic/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/1.8.0_212/25.212-b01]\r\n\u001b[7m^A^@^@^@^@^@^B\u001b(B\u001b[mV[2021-03-25T21:54:45,508][IN\u001b[1;30r\u001b[29;44H"] 107 | [65.903244, "o", "\u001b[1;29r\u001b[7S\u001b[22;44HFO ][o.e.n.Node ] JVM arguments [-Xms2g, -Xmx2g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFrac \rtion=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -Djdk.io.permissions \rUseCanonicalPath=true, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=fa \rlse, -Dlog4j2.disable.jmx=true, -Dlog4j.skipJansi=true, -XX:+HeapDumpOnOutOfMemoryError, -Des.path.home=/usr/share/elasticsearch]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mh[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [aggs-matrix-stats]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[md[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [ingest-common]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mf[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [lang-expression]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mb[2021-03-25T21:54:4"] 108 | [65.903565, "o", "6,166][INFO ][o.e.p.Plugin\u001b[1;30r\u001b[29;61H\u001b[1;29r\u001b[8S\u001b[21;61HsService ] [CizMOsQ] loaded module [lang-groovy]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[md[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [lang-mustache]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[md[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [lang-painless]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mb[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [parent-join]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[ma[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [percolator]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[m^[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [reindex]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mg[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [transport-netty3]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mg[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [transport-netty4]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mX[2021-03"] 109 | [65.903661, "o", "-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] no plugins lo\u001b[1;30r\u001b[29;99H\u001b[1;29r\u001b[5S\u001b[24;99Haded\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[ma[2021-03-25T21:54:47,634][INFO ][o.e.d.DiscoveryModule ] [CizMOsQ] using discovery type [zen]\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mH[2021-03-25T21:54:48,121][INFO ][o.e.n.Node ] initialized\r\n\u001b[7m^A^@^@^@^@^@^@\u001b(B\u001b[mS[2021-03-25T21:54:48,121][INFO ][o.e.n.Node ] [CizMOsQ] starting ...\r\n\u001b[7m^A^@^@^@^@^@^@<89>\u001b(B\u001b[m[2021-03-25T21:54:48,250][INFO ][o.e.t.TransportService ] [CizMOsQ] publish_address {127.0.0.1:9300}, bound_addresses {127.0.0.1:9300}\r\n\u001b[7mlogs/20210325225453-esbulk-test-es-5.6.16.log\u001b(B\u001b[m\u001b[K\u001b[1;30r\u001b[29;46H"] 110 | [67.179829, "o", "\r\u001b[K"] 111 | [67.188084, "o", "\u001b[?25l\u001b[H2021/03/25 22:54:51 created index: 200 OK\u001b[K\r\n2021/03/25 22:54:51 started 1 workers\u001b[K\r\n2021/03/25 22:54:51 on shutdown, number_of_replicas will be set back to 1\u001b[K\r\n2021/03/25 22:54:51 on shutdown, refresh_interval will be set back to 1s\u001b[K\r\n2021/03/25 22:54:51 applied setting: {\"index\": {\"refresh_interval\": \"-1\"}} with status 200 OK\u001b[K\r\n2021/03/25 22:54:51 start reading from fixtures/v10k.jsonl\u001b[K\r\n2021/03/25 22:54:51 message content-length will be 293890\u001b[K\r\n2021/03/25 22:54:52 [worker-0] @5000\u001b[K\r\n2021/03/25 22:54:52 message content-length will be 295000\u001b[K\r\n2021/03/25 22:54:52 [worker-0] @10000\u001b[K\r\n2021/03/25 22:54:52 10000 docs in 0.97s at 10350.515 docs/s with 1 workers\u001b[K\r\n2021/03/25 22:54:52 applied setting: {\"index\": {\"refresh_interval\": \"1s\"}} with status 200 OK\u001b[K\r\n2021/03/25 22:54:52 applied setting: {\"index\": {\"number_of_replicas\": \"1\"}} with status 200 OK\u001b[K\r\n2021/03/25 22:54:53 index flushed: 200 OK\u001b[K\r\n run_test.go:94: {\"took\":49,\"timed_out\":false,\"_shards\":{\"total\":5,\"successful\""] 112 | [67.188524, "o", ":5,\"skipped\":0,\"failed\":0},\"hits\":{\"total\":10000,\"max_score\":1.0,\"hits\":[{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma3u\",\"_score\":1.0,\"_source\":{\"v\": \"2\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma3v\",\"_score\":1.0,\"_source\":{\"v\": \"3\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma35\",\"_score\":1.0,\"_source\":{\"v\": \"13\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma38\",\"_score\":1.0,\"_source\":{\"v\": \"16\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4C\",\"_score\":1.0,\"_source\":{\"v\": \"22\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4E\",\"_score\":1.0,\"_source\":{\"v\": \"24\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4T\",\"_score\":1.0,\"_source\":{\"v\": \"39\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4X\",\"_score\":1.0,\"_source\":{\"v\": \"43\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4a\",\"_score\":1.0,\"_source\":{\"v\": \"46\"}},{\"_index\":\"abc\",\"_type\":\"any\",\"_id\":\"AXhrYhAMhIzWyFAmma4d\",\"_score\":1.0,\"_source\":{\"v\": \"49\"}}]}}\u001b"] 113 | [67.188817, "o", "[K\r\n run_test.go:178: es6 detected\u001b[K\r\n2021/03/25 22:54:53 logging to logs/20210325225453-esbulk-test-es-5.6.16.log\u001b[K\r\n--- PASS: TestMinimalConfig (45.40s)\u001b[K\r\nPASS\u001b[K\r\nok \u001b[4X\u001b[4Cgithub.com/miku/esbulk\u001b[2X\u001b[2C47.609s\u001b[K\r\n$ less logs/20210325225453-esbulk-test-es-5.6.16.log\u001b[K\r\n\"logs/20210325225453-esbulk-test-es-5.6.16.log\" may be a binary file. See it anyway? \u001b[K\r\n\u001b[K\u001b[1m\u001b[2m\r\n 0\u001b[38;5;250m:\u001b[38;5;255mbash\u001b[38;5;50m* \u001b(B\u001b[m\u001b[2m \u001b[38;5;233m\u001b[48;5;245m\u001b[1m 60% \u001b[48;5;241m 25/03 \u001b[48;5;245m 22:55 \u001b(B\u001b[m\u001b[29;1H\u001b[?12l\u001b[?25h"] 114 | [67.211549, "o", "$ "] 115 | [67.769563, "o", "less logs/20210325225453-esbulk-test-es-5.6.16.log"] 116 | [67.964756, "o", "\u001b[3G"] 117 | [68.2567, "o", "\u001b[C"] 118 | [68.414348, "o", "\u001b[C"] 119 | [68.553666, "o", "\u001b[C"] 120 | [68.683825, "o", "\u001b[C"] 121 | [68.833504, "o", "\u001b[C"] 122 | [69.122404, "o", "\u001b[4G\u001b[5Pogs/20210325225453-esbulk-test-es-5.6.16.log\u001b[3G"] 123 | [69.302083, "o", "clogs/20210325225453-esbulk-test-es-5.6.16.log\u001b[4G"] 124 | [69.411285, "o", "alogs/20210325225453-esbulk-test-es-5.6.16.log\u001b[5G"] 125 | [69.529548, "o", "tlogs/20210325225453-esbulk-test-es-5.6.16.log\u001b[6G"] 126 | [69.613847, "o", " logs/20210325225453-esbulk-test-es-5.6.16.log\u001b[7G"] 127 | [69.837288, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[1;30r\u001b[29;1H"] 128 | [69.848697, "o", "\u001b[1;29r\u001b[29S\u001b[HVM[Oracle Corporation/OpenJDK 64-Bit Server VM/1.8.0_212/25.212-b01]\r\nV[2021-03-25T21:54:45,508][INFO ][o.e.n.Node ] JVM arguments [-Xms2g, -Xmx2g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -Djdk.io.permissionsUseCanonicalPath=true, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Dlog4j.skipJansi=true, -XX:+HeapDumpOnOutOfMemoryError, -Des.path.home=/usr/share/elasticsearch]\r\nh[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [aggs-matrix-stats]\r\nd[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [ingest-common]\r\nf[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [lang-expression]\r\nb[2021-03-25T21:54:46,166]["] 129 | [69.849088, "o", "INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [lang-groovy]\r\nd[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [lang-mustache]\r\nd[2021-03-25T21:54:46,166][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [lang-painless]\r\nb[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [parent-join]\r\na[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [percolator]\r\n^[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [reindex]\r\ng[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [transport-netty3]\r\ng[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] loaded module [transport-netty4]\r\nX[2021-03-25T21:54:46,167][INFO ][o.e.p.PluginsService ] [CizMOsQ] no plugins loaded\r\na[2021-03-25T21:54:47,634][INFO ][o.e.d.DiscoveryModule ] [CizMOsQ] using discovery type [zen]\r\nH[2021-03-25T21:54:48,121][INFO ][o.e.n.Node "] 130 | [69.849451, "o", " ] initialized\r\nS[2021-03-25T21:54:48,121][INFO ][o.e.n.Node ] [CizMOsQ] starting ...\r\n[2021-03-25T21:54:48,250][INFO ][o.e.t.TransportService ] [CizMOsQ] publish_address {127.0.0.1:9300}, bound_addresses {127.0.0.1:9300}\r\n[2021-03-25T21:54:51,330][INFO ][o.e.c.s.ClusterService ] [CizMOsQ] new_master {CizMOsQ}{CizMOsQ0SkKB03BACJ6KWA}{Ue91QjnATlaw5XG8wCWsJQ}{127.0.0.1}{127.0.0.1:9300}, reason: zen-disco-elected-as-master ([0] nodes joined)\r\n[2021-03-25T21:54:51,366][INFO ][o.e.h.n.Netty4HttpServerTransport] [CizMOsQ] publish_address {172.17.0.3:9200}, bound_addresses {0.0.0.0:9200}\r\nN[2021-03-25T21:54:51,366][INFO ][o.e.n.Node ] [CizMOsQ] started\r\no[2021-03-25T21:54:51,395][INFO ][o.e.g.GatewayService ] [CizMOsQ] recovered [0] indices into cluster_state\r\n[2021-03-25T21:54:51,611][INFO ][o.e.c.m.MetaDataCreateIndexService] [CizMOsQ] [abc] creating index, cause [api], templates [], shards [5]/[1], mappings []\r\n|[2021-03-25T21:54:51,932][INFO ][o.e.c.s.IndexScopedSettings]"] 131 | [69.849741, "o", " [CizMOsQ] updating [index.refresh_interval] from [1s] to [-1]\u001b[1;30r\u001b[29;1H\u001b[1;29r\u001b[5S\u001b[24d|[2021-03-25T21:54:51,935][INFO ][o.e.c.s.IndexScopedSettings] [CizMOsQ] updating [index.refresh_interval] from [1s] to [-1]\r\n}[2021-03-25T21:54:52,155][INFO ][o.e.c.m.MetaDataMappingService] [CizMOsQ] [abc/JibqrNTXRvW4wh-mAW6P7A] create_mapping [any]\r\n|[2021-03-25T21:54:52,919][INFO ][o.e.c.s.IndexScopedSettings] [CizMOsQ] updating [index.refresh_interval] from [-1] to [1s]\r\n|[2021-03-25T21:54:52,920][INFO ][o.e.c.s.IndexScopedSettings] [CizMOsQ] updating [index.refresh_interval] from [-1] to [1s]\r\n[2021-03-25T21:54:52,935][INFO ][o.e.c.m.MetaDataUpdateSettingsService] [CizMOsQ] updating number_of_replicas to [1] for indices [abc]\u001b[1;30r\u001b[29;1H"] 132 | [69.873029, "o", "$ "] 133 | [71.694762, "o", "cat logs/20210325225453-esbulk-test-es-5.6.16.log"] 134 | [72.638337, "o", "\b\u001b[K"] 135 | [73.139157, "o", "\b\u001b[K"] 136 | [73.169078, "o", "\b\u001b[K"] 137 | [73.200105, "o", "\b\u001b[K"] 138 | [73.229721, "o", "\b\u001b[K"] 139 | [73.260206, "o", "\b\u001b[K"] 140 | [73.29042, "o", "\b\u001b[K"] 141 | [73.320691, "o", "\b\u001b[K"] 142 | [73.351279, "o", "\b\u001b[K"] 143 | [73.381551, "o", "\b\u001b[K"] 144 | [73.41127, "o", "\b\u001b[K"] 145 | [73.442231, "o", "\b\u001b[K"] 146 | [73.4728, "o", "\b\u001b[K"] 147 | [73.502011, "o", "\b\u001b[K"] 148 | [73.532284, "o", "\b\u001b[K"] 149 | [73.562842, "o", "\b\u001b[K"] 150 | [73.594502, "o", "\b\u001b[K"] 151 | [73.623528, "o", "\b\u001b[K"] 152 | [73.653688, "o", "\b\u001b[K"] 153 | [73.68349, "o", "\b\u001b[K"] 154 | [73.713614, "o", "\b\u001b[K"] 155 | [73.743588, "o", "\b\u001b[K"] 156 | [73.773984, "o", "\b\u001b[K"] 157 | [73.804756, "o", "\b\u001b[K"] 158 | [73.83514, "o", "\b\u001b[K"] 159 | [73.865198, "o", "\b\u001b[K"] 160 | [73.895458, "o", "\b\u001b[K"] 161 | [73.926127, "o", "\b\u001b[K"] 162 | [73.956476, "o", "\b\u001b[K"] 163 | [73.987208, "o", "\b\u001b[K"] 164 | [74.017487, "o", "\b\u001b[K"] 165 | [74.047435, "o", "\b\u001b[K"] 166 | [74.077529, "o", "\b\u001b[K"] 167 | [74.314122, "o", "\b\u001b[K"] 168 | [74.49093, "o", "\b\u001b[K"] 169 | [74.644746, "o", "\b\u001b[K"] 170 | [74.79802, "o", "\b\u001b[K"] 171 | [75.202314, "o", "\u001b[3G\u001b[K"] 172 | [76.036629, "o", "\u001b[1;29r\u001b[29;1H\n\u001b[28;3Hlogout\u001b[1;30r\u001b[29;1H"] 173 | [76.083689, "o", "\u001b[1;30r\u001b(B\u001b[m\u001b[?1l\u001b>\u001b[H\u001b[2J\u001b]112\u0007\u001b[?12l\u001b[?25h\u001b[?1000l\u001b[?1002l\u001b[?1006l\u001b[?1005l\u001b[?1004l"] 174 | [76.084353, "o", "[exited]\r\n"] 175 | [76.128861, "o", "tir@trieste:~/code/miku/esbulk \u001b[38;5;130m[git:master] \u001b[m$ "] 176 | [78.388365, "o", "e"] 177 | [78.604106, "o", "x"] 178 | [78.716566, "o", "i"] 179 | [78.829977, "o", "t"] 180 | [79.323252, "o", "\r\n"] 181 | [79.326385, "o", "exit\r\n"] 182 | -------------------------------------------------------------------------------- /docs/asciicast.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miku/esbulk/9984133e8f6c49fb1318686c0ff6a61107baab1c/docs/asciicast.gif -------------------------------------------------------------------------------- /docs/esbulk.1: -------------------------------------------------------------------------------- 1 | .TH ESBULK 1 "JANUAR 2018" "Leipzig University Library" "Manuals" 2 | .SH NAME 3 | .PP 4 | esbulk \- send documents to elasticsearch in bulk and in parallel. 5 | 6 | .SH SYNOPSIS 7 | .PP 8 | \fB\fCesbulk\fR [\fB\fC\-server\fR \fIURL\fP, \fB\fC\-index\fR \fIname\fP, \fB\fC\-size\fR \fIN\fP, \fB\fC\-w\fR \fIN\fP, \fB\fC\-z\fR] < \fIfile\fP 9 | 10 | .SH DESCRIPTION 11 | .PP 12 | esbulk takes as input a newline delimited JSON file and indexes all documents 13 | into elasticsearch running on a given server address. The documents are batched 14 | and indexed in parallel to achieve a high indexing throughput. 15 | 16 | .PP 17 | The newline delimited JSON text file format is explained at 18 | \[la]http://jsonlines.org/\[ra] and 19 | \[la]http://ndjson.org/\[ra]\&. 20 | 21 | .SH OPTIONS 22 | .PP 23 | \fB\fC\-0\fR 24 | Set the number of replicas to 0 during indexing (this can speed up indexing significantly, the original value is restored at the end and may cause delay until the cluster is green). 25 | 26 | .PP 27 | \fB\fC\-c\fR \fIstring\fP 28 | Create index mappings, settings, aliases, 29 | \[la]https://is.gd/3zszeu\[ra]\&. 30 | 31 | .PP 32 | \fB\fC\-cpuprofile\fR \fIstring\fP 33 | Write cpu profile to file. 34 | 35 | .PP 36 | \fB\fC\-id\fR \fIstring\fP 37 | Reuse value from this field as id. By default ids are autogenerated. 38 | 39 | .PP 40 | \fB\fC\-index\fR \fIstring\fP 41 | Index name. 42 | 43 | .PP 44 | \fB\fC\-mapping\fR \fIfilename\fP 45 | Mapping string or filename to apply before indexing. 46 | 47 | .PP 48 | \fB\fC\-memprofile\fR \fIstring\fP 49 | Write heap profile to file. 50 | 51 | .PP 52 | \fB\fC\-optype\fR \fIstring\fP 53 | optype (index \- will replace existing data, create \- will only create a new doc, 54 | update \- create new or update existing data) (default "index") 55 | 56 | .PP 57 | \fB\fC\-p\fR \fIname\fP 58 | Pipeline to use to preprocess documents. 59 | 60 | .PP 61 | \fB\fC\-purge\fR 62 | Purge any existing index before reindexing. Warning: No confirmation required. 63 | 64 | .PP 65 | \fB\fC\-r string\fR 66 | Refresh interval after import (default "1s") 67 | 68 | .PP 69 | \fB\fC\-server\fR \fIURL\fP 70 | Server hostport including schema like 71 | \[la]http://localhost:9200\[ra] 72 | 73 | .PP 74 | \fB\fC\-size\fR \fIN\fP 75 | Batch size. Defaults to 1000. Increase for small documents. 76 | 77 | .PP 78 | \fB\fC\-skipbroken\fR 79 | Skip broken json. 80 | 81 | .PP 82 | \fB\fC\-type\fR \fIstring\fP 83 | Elasticsearch type (deprecated in 6.0.0, 84 | \[la]https://is.gd/HFsOWt\[ra]), empty string. 85 | 86 | .PP 87 | \fB\fC\-u\fR \fIstring\fP 88 | HTTP basic authentication "username:password" (like curl \-u). 89 | 90 | .PP 91 | \fB\fC\-v\fR 92 | Program version. 93 | 94 | .PP 95 | \fB\fC\-verbose\fR 96 | Show progress. 97 | 98 | .PP 99 | \fB\fC\-w\fR \fIN\fP 100 | Number of workers. Defaults to number of cores. 101 | 102 | .PP 103 | \fB\fC\-z\fR 104 | Decompress gzip input file on the fly. 105 | 106 | .SH EXAMPLES 107 | .PP 108 | Index a compressed file: 109 | 110 | .PP 111 | \fB\fCesbulk \-index abc \-verbose \-server 110.81.131.200:9200 \-z file.ldj.gz\fR 112 | 113 | .PP 114 | Index from standard input: 115 | 116 | .PP 117 | \fB\fCcat file.ldj | esbulk \-index abc \-server 110.81.131.200:9200\fR 118 | 119 | .PP 120 | Purge an existing index, apply a mapping from a file and index: 121 | 122 | .PP 123 | \fB\fCesbulk \-purge \-mapping mapping.json \-index abc file.ldj\fR 124 | 125 | .SH DIAGNOSITCS 126 | .PP 127 | If indexing pressure on the bulk API is too high (dozens or hundreds of 128 | parallel workers, large batch sizes, depending on you setup), esbulk will halt 129 | and report an error: 130 | 131 | .PP 132 | .RS 133 | 134 | .nf 135 | $ esbulk \-index my\-index\-name \-w 100 file.ldj 136 | 2017/01/02 16:25:25 error during bulk operation, try less workers (lower \-w value) or 137 | increase thread\_pool.bulk.queue\_size in your nodes 138 | 139 | .fi 140 | .RE 141 | 142 | .PP 143 | Please note that, in such a case, some documents are indexed and some are not. 144 | Your index will be in an INCONSISTENT STATE, since there is no transactional 145 | bracket around the indexing process. 146 | 147 | .PP 148 | However, using defaults (parallism: number of cores) on a single node setup 149 | will just work. For larger clusters, increase the number of workers until you 150 | see full CPU utilization. After that, more workers won't buy any more speed. 151 | 152 | .SH BUGS 153 | .PP 154 | Please report bugs to 155 | \[la]https://github.com/miku/esbulk/issues\[ra]\&. 156 | 157 | .SH AUTHORS 158 | .PP 159 | Martin Czygan 160 | \[la]https://github.com/miku\[ra], 161 | \[la]martin.czygan@uni-leipzig.de\[ra] 162 | sakshambathla 163 | \[la]https://github.com/sakshambathla\[ra] 164 | Klaubert Herr 165 | \[la]https://github.com/klaubert\[ra] 166 | Yusuke KUOKA 167 | \[la]https://github.com/mumoshu\[ra] 168 | faultlin3 169 | \[la]https://github.com/faultlin3\[ra] 170 | gransy 171 | \[la]https://github.com/gransy\[ra] 172 | Christoph Kepper 173 | \[la]https://github.com/ckepper\[ra] 174 | Christian Solomon 175 | Mikael Byström 176 | 177 | .SH SEE ALSO 178 | .PP 179 | FINC 180 | \[la]https://finc.info\[ra], AMSL 181 | \[la]http://amsl.technology/\[ra], solrbulk 182 | \[la]https://github.com/miku/solrbulk\[ra] 183 | -------------------------------------------------------------------------------- /docs/esbulk.md: -------------------------------------------------------------------------------- 1 | ESBULK 1 "JANUAR 2018" "Leipzig University Library" "Manuals" 2 | ============================================================= 3 | 4 | NAME 5 | ---- 6 | 7 | esbulk - send documents to elasticsearch in bulk and in parallel. 8 | 9 | SYNOPSIS 10 | -------- 11 | 12 | `esbulk` [`-server` *URL*, `-index` *name*, `-size` *N*, `-w` *N*, `-z`] < *file* 13 | 14 | DESCRIPTION 15 | ----------- 16 | 17 | esbulk takes as input a newline delimited JSON file and indexes all documents 18 | into elasticsearch running on a given server address. The documents are batched 19 | and indexed in parallel to achieve a high indexing throughput. 20 | 21 | The newline delimited JSON text file format is explained at http://jsonlines.org/ and http://ndjson.org/. 22 | 23 | OPTIONS 24 | ------- 25 | 26 | `-0` 27 | Set the number of replicas to 0 during indexing (this can speed up indexing significantly, the original value is restored at the end and may cause delay until the cluster is green). 28 | 29 | `-c` *string* 30 | Create index mappings, settings, aliases, https://is.gd/3zszeu. 31 | 32 | `-cpuprofile` *string* 33 | Write cpu profile to file. 34 | 35 | `-id` *string* 36 | Reuse value from this field as id. By default ids are autogenerated. 37 | 38 | `-index` *string* 39 | Index name. 40 | 41 | `-mapping` *filename* 42 | Mapping string or filename to apply before indexing. 43 | 44 | `-memprofile` *string* 45 | Write heap profile to file. 46 | 47 | `-optype` *string* 48 | optype (index - will replace existing data, create - will only create a new doc, 49 | update - create new or update existing data) (default "index") 50 | 51 | `-p` *name* 52 | Pipeline to use to preprocess documents. 53 | 54 | `-purge` 55 | Purge any existing index before reindexing. Warning: No confirmation required. 56 | 57 | `-r string` 58 | Refresh interval after import (default "1s") 59 | 60 | `-server` *URL* 61 | Server hostport including schema like http://localhost:9200 62 | 63 | `-size` *N* 64 | Batch size. Defaults to 1000. Increase for small documents. 65 | 66 | `-skipbroken` 67 | Skip broken json. 68 | 69 | `-type` *string* 70 | Elasticsearch type (deprecated in 6.0.0, https://is.gd/HFsOWt), empty string. 71 | 72 | `-u` *string* 73 | HTTP basic authentication "username:password" (like curl -u). 74 | 75 | `-v` 76 | Program version. 77 | 78 | `-verbose` 79 | Show progress. 80 | 81 | `-w` *N* 82 | Number of workers. Defaults to number of cores. 83 | 84 | `-z` 85 | Decompress gzip input file on the fly. 86 | 87 | EXAMPLES 88 | -------- 89 | 90 | Index a compressed file: 91 | 92 | `esbulk -index abc -verbose -server 110.81.131.200:9200 -z file.ldj.gz` 93 | 94 | Index from standard input: 95 | 96 | `cat file.ldj | esbulk -index abc -server 110.81.131.200:9200` 97 | 98 | Purge an existing index, apply a mapping from a file and index: 99 | 100 | `esbulk -purge -mapping mapping.json -index abc file.ldj` 101 | 102 | DIAGNOSITCS 103 | ----------- 104 | 105 | If indexing pressure on the bulk API is too high (dozens or hundreds of 106 | parallel workers, large batch sizes, depending on you setup), esbulk will halt 107 | and report an error: 108 | 109 | ``` 110 | $ esbulk -index my-index-name -w 100 file.ldj 111 | 2017/01/02 16:25:25 error during bulk operation, try less workers (lower -w value) or 112 | increase thread_pool.bulk.queue_size in your nodes 113 | ``` 114 | 115 | Please note that, in such a case, some documents are indexed and some are not. 116 | Your index will be in an INCONSISTENT STATE, since there is no transactional 117 | bracket around the indexing process. 118 | 119 | However, using defaults (parallism: number of cores) on a single node setup 120 | will just work. For larger clusters, increase the number of workers until you 121 | see full CPU utilization. After that, more workers won't buy any more speed. 122 | 123 | BUGS 124 | ---- 125 | 126 | Please report bugs to https://github.com/miku/esbulk/issues. 127 | 128 | AUTHORS 129 | ------ 130 | 131 | Martin Czygan , 132 | sakshambathla 133 | Klaubert Herr 134 | Yusuke KUOKA 135 | faultlin3 136 | gransy 137 | Christoph Kepper 138 | Christian Solomon 139 | Mikael Byström 140 | 141 | SEE ALSO 142 | -------- 143 | 144 | [FINC](https://finc.info), [AMSL](http://amsl.technology/), [solrbulk](https://github.com/miku/solrbulk) 145 | 146 | -------------------------------------------------------------------------------- /docs/repo/authors.json: -------------------------------------------------------------------------------- 1 | {"y": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 25, 24, 24, 24, 24, 22, 21, 19], [310, 373, 379, 457, 457, 461, 461, 490, 491, 493, 563, 640, 642, 752, 752, 766, 886, 881, 905, 932, 927, 928, 950, 953, 1199, 1206, 1236, 1331, 1259], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 60, 60, 60, 60, 60, 60, 60, 58, 57]], "ts": ["2014-08-29T09:15:42", "2014-09-29T16:29:54", "2014-10-23T10:10:41", "2014-12-01T14:47:43", "2014-12-18T11:50:14", "2015-01-19T14:14:20", "2015-04-03T12:02:57", "2015-05-07T18:44:13", "2015-07-16T12:19:35", "2015-09-16T02:42:00", "2015-11-10T12:10:00", "2016-06-22T17:55:53", "2016-07-22T20:56:16", "2016-09-09T12:05:52", "2016-10-04T10:44:59", "2016-11-26T12:29:48", "2016-12-04T21:02:40", "2016-12-12T02:09:42", "2017-01-02T15:42:27", "2017-06-29T11:28:18", "2017-08-18T15:27:59", "2017-09-25T22:06:41", "2017-10-18T10:21:14", "2018-01-16T20:37:57", "2018-01-27T22:29:02", "2018-02-11T14:49:01", "2018-04-10T20:54:50", "2018-04-19T08:47:42", "2018-05-02T10:17:07"], "labels": ["Klaubert Herr", "Martin Czygan", "Yusuke KUOKA", "klaubert", "sakshambathla"]} -------------------------------------------------------------------------------- /docs/repo/authors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miku/esbulk/9984133e8f6c49fb1318686c0ff6a61107baab1c/docs/repo/authors.png -------------------------------------------------------------------------------- /docs/repo/cohorts.json: -------------------------------------------------------------------------------- 1 | {"y": [[310, 373, 379, 457, 457, 457, 457, 450, 448, 447, 412, 395, 395, 393, 393, 390, 378, 365, 365, 365, 365, 364, 364, 364, 363, 363, 355, 353, 343], [0, 0, 0, 0, 0, 4, 4, 40, 43, 46, 151, 141, 140, 140, 140, 140, 128, 128, 128, 128, 128, 128, 124, 124, 124, 124, 115, 115, 107], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 107, 219, 219, 236, 380, 388, 387, 368, 357, 356, 352, 352, 352, 352, 320, 312, 306], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 134, 166, 168, 197, 197, 194, 194, 192, 188, 183], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 253, 260, 339, 445, 512]], "ts": ["2014-08-29T09:15:42", "2014-09-29T16:29:54", "2014-10-23T10:10:41", "2014-12-01T14:47:43", "2014-12-18T11:50:14", "2015-01-19T14:14:20", "2015-04-03T12:02:57", "2015-05-07T18:44:13", "2015-07-16T12:19:35", "2015-09-16T02:42:00", "2015-11-10T12:10:00", "2016-06-22T17:55:53", "2016-07-22T20:56:16", "2016-09-09T12:05:52", "2016-10-04T10:44:59", "2016-11-26T12:29:48", "2016-12-04T21:02:40", "2016-12-12T02:09:42", "2017-01-02T15:42:27", "2017-06-29T11:28:18", "2017-08-18T15:27:59", "2017-09-25T22:06:41", "2017-10-18T10:21:14", "2018-01-16T20:37:57", "2018-01-27T22:29:02", "2018-02-11T14:49:01", "2018-04-10T20:54:50", "2018-04-19T08:47:42", "2018-05-02T10:17:07"], "labels": ["Code added in 2014", "Code added in 2015", "Code added in 2016", "Code added in 2017", "Code added in 2018"]} -------------------------------------------------------------------------------- /docs/repo/cohorts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miku/esbulk/9984133e8f6c49fb1318686c0ff6a61107baab1c/docs/repo/cohorts.png -------------------------------------------------------------------------------- /docs/repo/survival.json: -------------------------------------------------------------------------------- 1 | {"52234d8cc9a65877af89cef8acaf644a361d1abc": [[1409303742, 56], [1412008194, 33], [1414059041, 33], [1417445263, 16], [1418903414, 16], [1421676860, 16], [1428062577, 16], [1431024253, 16], [1437049175, 16], [1442371320, 16], [1447157400, 16], [1466618153, 16], [1469220976, 16], [1473422752, 16], [1475577899, 16], [1480163388, 16], [1480885360, 13], [1481508582, 10], [1483371747, 10], [1498735698, 10], [1503070079, 10], [1506377201, 10], [1508322074, 10], [1516135077, 10], [1517092142, 10], [1518360541, 10], [1523393690, 10], [1524127662, 10], [1525256227, 10]], "4b5f141d4294b4238e0bbdb74e1ef4f1bb220afc": [[1412008194, 20], [1414059041, 16], [1417445263, 16], [1418903414, 16], [1421676860, 16], [1428062577, 16], [1431024253, 16], [1437049175, 16], [1442371320, 16], [1447157400, 16], [1466618153, 7], [1469220976, 7], [1473422752, 7], [1475577899, 7], [1480163388, 7], [1480885360, 7], [1481508582, 7], [1483371747, 7], [1498735698, 7], [1503070079, 7], [1506377201, 7], [1508322074, 7], [1516135077, 7], [1517092142, 7], [1518360541, 7], [1523393690, 7], [1524127662, 7], [1525256227, 7]], "4f1641d6cd631f034cd927991c08df53efb770d3": [[1412008194, 9], [1414059041, 9], [1417445263, 2], [1418903414, 2], [1421676860, 2], [1428062577, 2], [1431024253, 2], [1437049175, 2], [1442371320, 2], [1447157400, 2], [1466618153, 1], [1469220976, 1], [1473422752, 1], [1475577899, 1], [1480163388, 1]], "92a1cd9377fa178d576c2f04f01dffdb35c7803f": [[1412008194, 3]], "2865bfbbb69c30a20407d2ac236e7cea1cc9e729": [[1412008194, 3], [1414059041, 3], [1417445263, 3], [1418903414, 3], [1421676860, 3], [1428062577, 3], [1431024253, 3], [1437049175, 3], [1442371320, 3], [1447157400, 3], [1466618153, 3], [1469220976, 3], [1473422752, 3], [1475577899, 3], [1480163388, 3], [1480885360, 3], [1481508582, 3], [1483371747, 3], [1498735698, 3], [1503070079, 3], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "9bfaa92544eaefcbc2869b8f7a804b3e4b37bb93": [[1412008194, 16], [1414059041, 16], [1417445263, 2], [1418903414, 2], [1421676860, 2], [1428062577, 2], [1431024253, 2], [1437049175, 2], [1442371320, 2], [1447157400, 2], [1466618153, 2], [1469220976, 2], [1473422752, 2], [1475577899, 2], [1480163388, 2], [1480885360, 2], [1481508582, 2], [1483371747, 2], [1498735698, 2], [1503070079, 2], [1506377201, 2], [1508322074, 2], [1516135077, 2], [1517092142, 2], [1518360541, 2], [1523393690, 2], [1524127662, 2], [1525256227, 2]], "4aed98ec0d1db3f2252c29c33b04d5dbd90ca94d": [[1412008194, 2], [1414059041, 2]], "7e7c6e4216cbed73b899938daf1a0aefb9b0d711": [[1412008194, 1], [1414059041, 1]], "78720d0fecea1d4de2c258bc1129b8a4be8bcc2b": [[1412008194, 11], [1414059041, 11], [1417445263, 10], [1418903414, 10], [1421676860, 10], [1428062577, 10], [1431024253, 10], [1437049175, 10], [1442371320, 10], [1447157400, 10], [1466618153, 10], [1469220976, 10], [1473422752, 10], [1475577899, 10], [1480163388, 10], [1480885360, 10], [1481508582, 10], [1483371747, 10], [1498735698, 10], [1503070079, 10], [1506377201, 10], [1508322074, 10], [1516135077, 10], [1517092142, 10], [1518360541, 10], [1523393690, 10], [1524127662, 10], [1525256227, 10]], "fc8b9af6c1920182f337901d0cd0d31de35975f2": [[1412008194, 3], [1414059041, 3]], "f81871f21e999b9e3f3d8ff3de372cfc9fa6fb6d": [[1412008194, 3], [1414059041, 3]], "5c9d549bdf8f9b2be9b5de912b51a85a4dd1859e": [[1412008194, 14], [1414059041, 14], [1417445263, 5], [1418903414, 5], [1421676860, 5], [1428062577, 5], [1431024253, 4], [1437049175, 4], [1442371320, 4], [1447157400, 4], [1466618153, 4], [1469220976, 4], [1473422752, 4], [1475577899, 4], [1480163388, 4], [1480885360, 4], [1481508582, 4], [1483371747, 4], [1498735698, 4], [1503070079, 4], [1506377201, 4], [1508322074, 4], [1516135077, 4], [1517092142, 4], [1518360541, 4], [1523393690, 4], [1524127662, 4], [1525256227, 1]], "e0df8880adaf20c6453daff871c7bc432960b431": [[1412008194, 4], [1414059041, 4]], "aaf74d3925c8b4f70284e83749cd50a8a5a501b7": [[1414059041, 2], [1417445263, 2], [1418903414, 2], [1421676860, 2], [1428062577, 2], [1431024253, 2], [1437049175, 2], [1442371320, 2], [1447157400, 2], [1466618153, 2], [1469220976, 2], [1473422752, 2], [1475577899, 2], [1480163388, 2], [1480885360, 2], [1481508582, 2], [1483371747, 2], [1498735698, 2], [1503070079, 2], [1506377201, 2], [1508322074, 2], [1516135077, 2], [1517092142, 2], [1518360541, 2], [1523393690, 2], [1524127662, 2], [1525256227, 2]], "44bba11fd9de8e60f9dd1243f36163f80be0a5b2": [[1414059041, 1], [1417445263, 1], [1418903414, 1], [1421676860, 1], [1428062577, 1], [1431024253, 1], [1437049175, 1], [1442371320, 1], [1447157400, 1]], "009841457645c7be536cc73aa0b939349b51be67": [[1414059041, 13], [1417445263, 9], [1418903414, 9], [1421676860, 9], [1428062577, 9], [1431024253, 9], [1437049175, 9], [1442371320, 9], [1447157400, 9], [1466618153, 8], [1469220976, 8], [1473422752, 8], [1475577899, 8], [1480163388, 8], [1480885360, 8], [1481508582, 8], [1483371747, 8], [1498735698, 8], [1503070079, 8], [1506377201, 8], [1508322074, 8], [1516135077, 8], [1517092142, 8], [1518360541, 8], [1523393690, 8], [1524127662, 8], [1525256227, 8]], "c972203344618a28ebccc858a67456ada1fd544a": [[1417445263, 1], [1418903414, 1], [1421676860, 1], [1428062577, 1], [1431024253, 1]], "233d3a5f773ed0e01a8f1385e3fb61c23b116075": [[1417445263, 8], [1418903414, 8], [1421676860, 8], [1428062577, 8], [1431024253, 8], [1437049175, 8], [1442371320, 8], [1447157400, 8], [1466618153, 4], [1469220976, 4], [1473422752, 3], [1475577899, 3], [1480163388, 3], [1480885360, 3], [1481508582, 3], [1483371747, 3], [1498735698, 3], [1503070079, 3], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "9f726c7f693898b8f3feffcf193314abac09217c": [[1417445263, 64], [1418903414, 64], [1421676860, 64], [1428062577, 64], [1431024253, 64], [1437049175, 64], [1442371320, 63], [1447157400, 58], [1466618153, 58], [1469220976, 58], [1473422752, 57], [1475577899, 57], [1480163388, 54], [1480885360, 49], [1481508582, 49], [1483371747, 49], [1498735698, 49], [1503070079, 49], [1506377201, 49], [1508322074, 49], [1516135077, 49], [1517092142, 49], [1518360541, 49], [1523393690, 49], [1524127662, 47], [1525256227, 47]], "8a8a121fb1abf6a989a128c1f955d6ee638339a6": [[1417445263, 7], [1418903414, 7], [1421676860, 7], [1428062577, 7], [1431024253, 4], [1437049175, 4], [1442371320, 4], [1447157400, 4], [1466618153, 4], [1469220976, 4], [1473422752, 4], [1475577899, 4], [1480163388, 4], [1480885360, 4], [1481508582, 4], [1483371747, 4], [1498735698, 4], [1503070079, 4], [1506377201, 4], [1508322074, 4], [1516135077, 4], [1517092142, 4], [1518360541, 4], [1523393690, 4], [1524127662, 4], [1525256227, 4]], "a65294f7934a863229fc114ae71a322614a31b7e": [[1417445263, 7], [1418903414, 7], [1421676860, 7], [1428062577, 7], [1431024253, 7], [1437049175, 7], [1442371320, 7], [1447157400, 7], [1466618153, 7], [1469220976, 7], [1473422752, 7], [1475577899, 7], [1480163388, 7], [1480885360, 7], [1481508582, 7], [1483371747, 7], [1498735698, 7], [1503070079, 7], [1506377201, 7], [1508322074, 7], [1516135077, 7], [1517092142, 7], [1518360541, 7], [1523393690, 7], [1524127662, 7], [1525256227, 7]], "6c2007667489141bf71878b46c174f8d013f8b90": [[1417445263, 1], [1418903414, 1], [1421676860, 1], [1428062577, 1], [1431024253, 1], [1437049175, 1], [1442371320, 1], [1447157400, 1], [1466618153, 1], [1469220976, 1], [1473422752, 1], [1475577899, 1], [1480163388, 1]], "69f01b2ed0ada30c6656793ff3cf2f5f227afbb8": [[1417445263, 4], [1418903414, 4], [1421676860, 4], [1428062577, 4], [1431024253, 4], [1437049175, 4], [1442371320, 4], [1447157400, 4], [1466618153, 4], [1469220976, 4], [1473422752, 4], [1475577899, 4], [1480163388, 4], [1480885360, 4], [1481508582, 4], [1483371747, 4], [1498735698, 4], [1503070079, 4], [1506377201, 4], [1508322074, 4], [1516135077, 4], [1517092142, 4], [1518360541, 4], [1523393690, 4], [1524127662, 4], [1525256227, 4]], "58d3d9df2cbed9f10ec9f4d69db13e59b61a4b44": [[1417445263, 5], [1418903414, 5], [1421676860, 5], [1428062577, 5], [1431024253, 5], [1437049175, 5], [1442371320, 5], [1447157400, 5], [1466618153, 5], [1469220976, 5], [1473422752, 5], [1475577899, 5], [1480163388, 5], [1480885360, 5], [1481508582, 5], [1483371747, 5], [1498735698, 5], [1503070079, 5], [1506377201, 5], [1508322074, 5], [1516135077, 5], [1517092142, 5], [1518360541, 5], [1523393690, 5], [1524127662, 5], [1525256227, 5]], "89a1fd8b1289de80705a11214912050df39058db": [[1417445263, 9], [1418903414, 9], [1421676860, 9], [1428062577, 9], [1431024253, 9], [1437049175, 9], [1442371320, 9], [1447157400, 9], [1466618153, 9], [1469220976, 9], [1473422752, 9], [1475577899, 9], [1480163388, 9], [1480885360, 9], [1481508582, 9], [1483371747, 9], [1498735698, 9], [1503070079, 9], [1506377201, 9], [1508322074, 9], [1516135077, 9], [1517092142, 9], [1518360541, 9], [1523393690, 9], [1524127662, 9], [1525256227, 9]], "e4b6d522ac5ce8438fd3cabdaff10b71f4325126": [[1417445263, 12], [1418903414, 12], [1421676860, 12], [1428062577, 12], [1431024253, 12], [1437049175, 12], [1442371320, 12], [1447157400, 12], [1466618153, 12], [1469220976, 12], [1473422752, 12], [1475577899, 12], [1480163388, 12], [1480885360, 12], [1481508582, 12], [1483371747, 12], [1498735698, 12], [1503070079, 12], [1506377201, 12], [1508322074, 12], [1516135077, 12], [1517092142, 12], [1518360541, 12], [1523393690, 12], [1524127662, 12], [1525256227, 12]], "01372fbc4adae64f0c14fcdaf4dc80d756c1a75c": [[1417445263, 11], [1418903414, 11], [1421676860, 11], [1428062577, 11], [1431024253, 11], [1437049175, 11], [1442371320, 11], [1447157400, 11], [1466618153, 11], [1469220976, 11], [1473422752, 11], [1475577899, 11], [1480163388, 11], [1480885360, 11], [1481508582, 11], [1483371747, 11], [1498735698, 11], [1503070079, 11], [1506377201, 10], [1508322074, 10], [1516135077, 10], [1517092142, 10], [1518360541, 10], [1523393690, 10], [1524127662, 10], [1525256227, 10]], "2ce69446860cae39768bbabfaad3d2faf9c6d1b1": [[1417445263, 1], [1418903414, 1], [1421676860, 1], [1428062577, 1], [1431024253, 1], [1437049175, 1], [1442371320, 1], [1447157400, 1], [1466618153, 1], [1469220976, 1], [1473422752, 1], [1475577899, 1], [1480163388, 1], [1480885360, 1], [1481508582, 1], [1483371747, 1], [1498735698, 1], [1503070079, 1], [1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "b77c307a82e137ddbdd2c31dec5e52aa43e7c8cc": [[1417445263, 35], [1418903414, 35], [1421676860, 35], [1428062577, 35], [1431024253, 33], [1437049175, 33], [1442371320, 33], [1447157400, 17], [1466618153, 16], [1469220976, 16], [1473422752, 16], [1475577899, 16], [1480163388, 16], [1480885360, 15], [1481508582, 15], [1483371747, 15], [1498735698, 15], [1503070079, 15], [1506377201, 15], [1508322074, 15], [1516135077, 15], [1517092142, 15], [1518360541, 15], [1523393690, 7], [1524127662, 7]], "121d473b668a6f196fdaf2779b81c33ec91a0009": [[1417445263, 3], [1418903414, 3], [1421676860, 3], [1428062577, 3], [1431024253, 2], [1437049175, 2], [1442371320, 2]], "71ec563dc7de31be9e528a2cb4f38adcb119ba69": [[1421676860, 4], [1428062577, 4], [1431024253, 4], [1437049175, 4], [1442371320, 4], [1447157400, 4], [1466618153, 4], [1469220976, 4], [1473422752, 4], [1475577899, 4], [1480163388, 4], [1480885360, 4], [1481508582, 4], [1483371747, 4], [1498735698, 4], [1503070079, 4], [1506377201, 4], [1508322074, 4], [1516135077, 4], [1517092142, 4], [1518360541, 4], [1523393690, 4], [1524127662, 4], [1525256227, 4]], "6597fdf0e281ea15c66c6c06d18b0a4d7fa21df9": [[1431024253, 6], [1437049175, 6], [1442371320, 6], [1447157400, 3], [1466618153, 3], [1469220976, 3], [1473422752, 3], [1475577899, 3], [1480163388, 3], [1480885360, 3], [1481508582, 3], [1483371747, 3], [1498735698, 3], [1503070079, 3], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "79da0a931a309526e94ad3dd9963a246bf2c40f0": [[1431024253, 1], [1437049175, 1], [1442371320, 1], [1447157400, 1], [1466618153, 1], [1469220976, 1], [1473422752, 1], [1475577899, 1], [1480163388, 1], [1480885360, 1], [1481508582, 1], [1483371747, 1], [1498735698, 1], [1503070079, 1], [1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "6cfe8affa45aa84cb7519596b56e44e9d587a5e4": [[1431024253, 29], [1437049175, 29], [1442371320, 29], [1447157400, 11], [1466618153, 11], [1469220976, 11], [1473422752, 11], [1475577899, 11], [1480163388, 11], [1480885360, 11], [1481508582, 11], [1483371747, 11], [1498735698, 11], [1503070079, 11], [1506377201, 11], [1508322074, 11], [1516135077, 11], [1517092142, 11], [1518360541, 11], [1523393690, 7], [1524127662, 7], [1525256227, 4]], "d7c36ddd422c4c34043018b4513ef577cde12612": [[1437049175, 2], [1442371320, 2], [1447157400, 1], [1466618153, 1], [1469220976, 1], [1473422752, 1], [1475577899, 1], [1480163388, 1], [1480885360, 1], [1481508582, 1], [1483371747, 1], [1498735698, 1], [1503070079, 1], [1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "882f3acd8d8c821a5f841f0ae70b980b9bb7f189": [[1437049175, 1], [1442371320, 1], [1447157400, 1], [1466618153, 1]], "d9fb9d408b32e17cdfd21e4fa5420d20f5e37c59": [[1442371320, 3]], "439ea7f1be680a7b56db6e7effdf262897f7f48f": [[1447157400, 11], [1466618153, 11], [1469220976, 11], [1473422752, 11], [1475577899, 11], [1480163388, 11], [1480885360, 11], [1481508582, 11], [1483371747, 11], [1498735698, 11], [1503070079, 11], [1506377201, 11], [1508322074, 11], [1516135077, 11], [1517092142, 11], [1518360541, 11], [1523393690, 11], [1524127662, 11], [1525256227, 11]], "1eabed72590847fbec5f2f79c9d0881c564fa08e": [[1447157400, 115], [1466618153, 108], [1469220976, 108], [1473422752, 108], [1475577899, 108], [1480163388, 108], [1480885360, 96], [1481508582, 96], [1483371747, 96], [1498735698, 96], [1503070079, 96], [1506377201, 96], [1508322074, 92], [1516135077, 92], [1517092142, 92], [1518360541, 92], [1523393690, 87], [1524127662, 87], [1525256227, 82]], "6292c1dc76cc2a286f75276a738082e155fdc42c": [[1447157400, 4], [1466618153, 1], [1469220976, 1], [1473422752, 1], [1475577899, 1], [1480163388, 1], [1480885360, 1], [1481508582, 1], [1483371747, 1], [1498735698, 1], [1503070079, 1], [1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "0aefba0d42f51dee4610d1c0c14d2964b2e76385": [[1466618153, 92], [1469220976, 92], [1473422752, 76], [1475577899, 76], [1480163388, 72], [1480885360, 66], [1481508582, 66], [1483371747, 66], [1498735698, 66], [1503070079, 65], [1506377201, 65], [1508322074, 65], [1516135077, 65], [1517092142, 65], [1518360541, 65], [1523393690, 61], [1524127662, 57], [1525256227, 57]], "6bc78ee9ed0df96134c8b6135c8f8a884621469a": [[1466618153, 1], [1469220976, 1], [1473422752, 1], [1475577899, 1], [1480163388, 1], [1480885360, 1], [1481508582, 1], [1483371747, 1], [1498735698, 1], [1503070079, 1], [1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "67899e5e639ee0a88878f9cfb6206371d2edb6df": [[1466618153, 3], [1469220976, 3]], "f38053c38eb84d3bf56f9557ffd2b84c05c553b7": [[1466618153, 8], [1469220976, 8], [1473422752, 8], [1475577899, 8], [1480163388, 8], [1480885360, 4], [1481508582, 4], [1483371747, 4], [1498735698, 4], [1503070079, 4], [1506377201, 4], [1508322074, 4], [1516135077, 4], [1517092142, 4], [1518360541, 4], [1523393690, 4], [1524127662, 3], [1525256227, 3]], "66bfb9950e8f06757435831ff10b3031bac1f6c4": [[1469220976, 1], [1473422752, 1], [1475577899, 1], [1480163388, 1], [1480885360, 1], [1481508582, 1]], "9b4c5914e490a0a445197e677ca6773f5bbe099d": [[1469220976, 2], [1473422752, 2], [1475577899, 2], [1480163388, 2], [1480885360, 2], [1481508582, 2], [1483371747, 2], [1498735698, 2], [1503070079, 2], [1506377201, 2], [1508322074, 2], [1516135077, 2], [1517092142, 2], [1518360541, 2], [1523393690, 2], [1524127662, 2], [1525256227, 2]], "3e095823cfdfbc70e3aa973ae045e4ad7eba23e6": [[1473422752, 64], [1475577899, 64], [1480163388, 64], [1480885360, 48], [1481508582, 48], [1483371747, 48], [1498735698, 48], [1503070079, 48], [1506377201, 48], [1508322074, 48], [1516135077, 48], [1517092142, 48], [1518360541, 48], [1523393690, 48], [1524127662, 48], [1525256227, 48]], "5761fbe35b559a3553db58564b6e23789da6378d": [[1473422752, 5], [1475577899, 5], [1480163388, 5], [1480885360, 5], [1481508582, 5], [1483371747, 5], [1498735698, 5], [1503070079, 5], [1506377201, 5], [1508322074, 5], [1516135077, 5], [1517092142, 5], [1518360541, 5], [1523393690, 5], [1524127662, 5], [1525256227, 5]], "2712a66fa2822e673c1e8f294f4bff49aad2b3dd": [[1473422752, 30], [1475577899, 30], [1480163388, 30], [1480885360, 30], [1481508582, 30], [1483371747, 30], [1498735698, 25], [1503070079, 24], [1506377201, 23], [1508322074, 23], [1516135077, 23], [1517092142, 23], [1518360541, 23], [1523393690, 23], [1524127662, 23], [1525256227, 22]], "201490f089028d2e5536f6014b50378d709c613f": [[1473422752, 3], [1475577899, 3]], "55291757ef1fa45f859bf52e76140e3e84f26978": [[1473422752, 2], [1475577899, 2], [1480163388, 2], [1480885360, 2], [1481508582, 2], [1483371747, 2], [1498735698, 2]], "5a29322f4ac325eb5cabf81548b088afbe724195": [[1473422752, 3], [1475577899, 3], [1480163388, 3], [1480885360, 3], [1481508582, 3], [1483371747, 3], [1498735698, 3], [1503070079, 3], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "0bbffb3f0966b066f739394965185aade01ebbdd": [[1473422752, 21], [1475577899, 21], [1480163388, 20], [1480885360, 20], [1481508582, 20], [1483371747, 20], [1498735698, 9], [1503070079, 8], [1506377201, 8], [1508322074, 8], [1516135077, 8], [1517092142, 8], [1518360541, 8], [1523393690, 8], [1524127662, 8], [1525256227, 8]], "a85c5b5d6587f2c90fbe5363bddd8723b18aedaf": [[1473422752, 3], [1475577899, 3], [1480163388, 3], [1480885360, 2], [1481508582, 2], [1483371747, 2], [1498735698, 2], [1503070079, 2], [1506377201, 2], [1508322074, 2], [1516135077, 2], [1517092142, 2], [1518360541, 2], [1523393690, 2], [1524127662, 2], [1525256227, 2]], "fd291a9a7e81761cc04e68ad08c334ce66920b5d": [[1480163388, 7], [1480885360, 3], [1481508582, 3], [1483371747, 3], [1498735698, 3], [1503070079, 3], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "66d43cd61149847e8d92eefe3e7be9ed7b2572c1": [[1480163388, 7], [1480885360, 6], [1481508582, 6], [1483371747, 6], [1498735698, 6], [1503070079, 6], [1506377201, 6], [1508322074, 6], [1516135077, 6], [1517092142, 6], [1518360541, 6], [1523393690, 6], [1524127662, 6], [1525256227, 6]], "99218a6f8b048c06ec214d3cd54cd87614f05692": [[1480163388, 11], [1480885360, 7], [1481508582, 7], [1483371747, 7], [1498735698, 7], [1503070079, 7], [1506377201, 7], [1508322074, 7], [1516135077, 7], [1517092142, 7], [1518360541, 7], [1523393690, 7], [1524127662, 7], [1525256227, 7]], "5a313da3e2fdd77ed64c0c8b7b8c37aa16dad74b": [[1480885360, 15], [1481508582, 15], [1483371747, 15], [1498735698, 15], [1503070079, 15], [1506377201, 15], [1508322074, 15], [1516135077, 15], [1517092142, 15], [1518360541, 15]], "96b9452602652ee9c0688706c70c51f48f151fcb": [[1480885360, 3], [1481508582, 3], [1483371747, 3], [1498735698, 3], [1503070079, 3], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "19ec9dde6bd069b9a2b4394bbddd3bddfe079b6d": [[1480885360, 9], [1481508582, 9], [1483371747, 9], [1498735698, 9], [1503070079, 9], [1506377201, 9], [1508322074, 9], [1516135077, 9], [1517092142, 9], [1518360541, 9], [1523393690, 9], [1524127662, 9], [1525256227, 9]], "b16722f3e733f579b4a2ca0619500f1cd6044916": [[1480885360, 44], [1481508582, 44], [1483371747, 44], [1498735698, 44], [1503070079, 40], [1506377201, 40], [1508322074, 40], [1516135077, 40], [1517092142, 40], [1518360541, 40], [1523393690, 40], [1524127662, 40], [1525256227, 40]], "7ff9edfc9ae9241b1ab1651fec97bda555c236d5": [[1480885360, 8], [1481508582, 8], [1483371747, 8], [1498735698, 8], [1503070079, 7], [1506377201, 7], [1508322074, 7], [1516135077, 7], [1517092142, 7], [1518360541, 7], [1523393690, 6], [1524127662, 5], [1525256227, 5]], "057230970bd55a7a1cef6ac118bfb148bb908ffb": [[1480885360, 71], [1481508582, 71], [1483371747, 71], [1498735698, 71], [1503070079, 70], [1506377201, 70], [1508322074, 66], [1516135077, 66], [1517092142, 66], [1518360541, 66], [1523393690, 56], [1524127662, 54], [1525256227, 51]], "065ceb57f3820e7d30eed4c99c13b1923bacbd68": [[1480885360, 6], [1481508582, 6], [1483371747, 6], [1498735698, 6], [1503070079, 6], [1506377201, 6], [1508322074, 6], [1516135077, 6], [1517092142, 6], [1518360541, 6], [1523393690, 6], [1524127662, 6], [1525256227, 6]], "da7fd518029dc2ac69f7936d4f77f5d39dfb5c7c": [[1480885360, 2], [1481508582, 2], [1483371747, 2], [1498735698, 2], [1503070079, 2], [1506377201, 2], [1508322074, 2], [1516135077, 2], [1517092142, 2], [1518360541, 2], [1523393690, 2], [1524127662, 2], [1525256227, 2]], "1eeea56f3d22f45d88464dd6cb2f3ece2826fca1": [[1480885360, 1], [1481508582, 1], [1483371747, 1], [1498735698, 1], [1503070079, 1], [1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "9b51c7203cd6d28c8f957aede002119b0be883e2": [[1480885360, 7], [1481508582, 7], [1483371747, 7], [1498735698, 4], [1503070079, 4], [1506377201, 4], [1508322074, 4], [1516135077, 4], [1517092142, 4], [1518360541, 4], [1523393690, 4], [1524127662, 4], [1525256227, 4]], "b720bfb12aae08599df77bf5145e4ce2a0167bc5": [[1480885360, 2], [1481508582, 2], [1483371747, 2], [1498735698, 2], [1503070079, 2], [1506377201, 2], [1508322074, 2], [1516135077, 2], [1517092142, 2], [1518360541, 2], [1523393690, 2], [1524127662, 2], [1525256227, 2]], "c2567a866721a3a639c4748edc22c10d4d6ad2a7": [[1480885360, 2], [1481508582, 2], [1483371747, 2], [1498735698, 2], [1503070079, 2], [1506377201, 2], [1508322074, 2], [1516135077, 2], [1517092142, 2], [1518360541, 2], [1523393690, 2], [1524127662, 2], [1525256227, 2]], "e5d0a0e07365c4724587e04e3a0a44ab47dbec97": [[1480885360, 1]], "c846def538bab54c0a9df39b6892ce447f2de56b": [[1480885360, 4], [1481508582, 4], [1483371747, 4], [1498735698, 4], [1503070079, 4], [1506377201, 4], [1508322074, 4], [1516135077, 4], [1517092142, 4], [1518360541, 4], [1523393690, 4], [1524127662, 4], [1525256227, 2]], "3c9c32e287c1f9f86c631700c0b9c12429bb719c": [[1480885360, 5], [1481508582, 5], [1483371747, 5], [1498735698, 5], [1503070079, 5], [1506377201, 5], [1508322074, 5], [1516135077, 5], [1517092142, 5], [1518360541, 5], [1523393690, 5], [1524127662, 5], [1525256227, 5]], "52339249b8b3ec48b50ca01fb238e9085b9658cf": [[1481508582, 9], [1483371747, 9], [1498735698, 9], [1503070079, 9], [1506377201, 9], [1508322074, 9], [1516135077, 9], [1517092142, 9], [1518360541, 9], [1523393690, 9], [1524127662, 9], [1525256227, 9]], "8a41cbc2a17636bc082aee935a7d9d359872c225": [[1483371747, 21], [1498735698, 21], [1503070079, 21], [1506377201, 21], [1508322074, 21], [1516135077, 21], [1517092142, 21], [1518360541, 21], [1523393690, 21], [1524127662, 21], [1525256227, 21]], "160b7b21d621eefe7909d20031f0579a25fd6c4c": [[1483371747, 1], [1498735698, 1], [1503070079, 1], [1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "48eda37a9cf6926b9c290bf50e3df3cfa4bdbcff": [[1483371747, 3], [1498735698, 3], [1503070079, 3], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "4c5c9b4e672710f9194bbf2eaec0e264bb10c041": [[1498735698, 43], [1503070079, 43], [1506377201, 43], [1508322074, 43], [1516135077, 43], [1517092142, 43], [1518360541, 43], [1523393690, 43], [1524127662, 43], [1525256227, 43]], "08f7e429d230a9f2319ecf82d254690efdd269ec": [[1498735698, 63], [1503070079, 60], [1506377201, 60], [1508322074, 60], [1516135077, 60], [1517092142, 60], [1518360541, 60], [1523393690, 60], [1524127662, 58], [1525256227, 57]], "abbd6af4a0d38de5ad3af105791c76c20cc69ef2": [[1498735698, 3]], "206144894aa7a25a996666a8b0d58c42289fcae5": [[1503070079, 7], [1506377201, 7], [1508322074, 7], [1516135077, 7], [1517092142, 7], [1518360541, 7], [1523393690, 7], [1524127662, 7], [1525256227, 5]], "8fd99da7648093b32cf6cadaf0346825f346dcd8": [[1503070079, 22], [1506377201, 22], [1508322074, 21], [1516135077, 21], [1517092142, 21], [1518360541, 21], [1523393690, 21], [1524127662, 20], [1525256227, 19]], "8f74220a3eb41d57d4cd9c73a15f7310267a7a6d": [[1503070079, 3], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "7cfa95b0a22b96699bfc1bfabc33c35662363d21": [[1503070079, 4], [1506377201, 3], [1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 1], [1524127662, 1]], "55fc4fb1b62be15a4414dad8e51cee6b058424c6": [[1503070079, 2]], "1ef78156a1365d10ea1a432f598c42802d332ae5": [[1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "a6b167e4ccb99bb5524ae46f3ceeef9eb43e6e8e": [[1506377201, 3]], "0e06857a85bb9ef079dc4a4d64c164464d837d46": [[1506377201, 1], [1508322074, 1], [1516135077, 1], [1517092142, 1], [1518360541, 1], [1523393690, 1], [1524127662, 1], [1525256227, 1]], "71539effa8e9ef638c7a1230d7603e61492d6d98": [[1508322074, 10], [1516135077, 10], [1517092142, 10], [1518360541, 10], [1523393690, 10], [1524127662, 9], [1525256227, 9]], "b406b18b081aec89f714063df4593a3c9cb42b57": [[1508322074, 9], [1516135077, 9], [1517092142, 9], [1518360541, 9], [1523393690, 9], [1524127662, 9], [1525256227, 9]], "e4d7b15d32d8c783bccd8e56ae6f60833bca15c4": [[1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "eeef00be42f4b131bf07df7abf936f7bf13dfbd3": [[1508322074, 5], [1516135077, 5], [1517092142, 5], [1518360541, 5], [1523393690, 5], [1524127662, 5], [1525256227, 5]], "0ec345c494282a9b63d4ba2c89123e4a4dffa4b8": [[1508322074, 3], [1516135077, 3]], "663f5f230a9ec0512030ff3f9b2ad19385965479": [[1508322074, 3], [1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "0142a940879b0bf7938fbdf3eed1185edaccb124": [[1516135077, 3], [1517092142, 3], [1518360541, 3], [1523393690, 3], [1524127662, 3], [1525256227, 3]], "37e25dc801bfd3066f8c5da3d85dc676e78f73c2": [[1517092142, 235], [1518360541, 235], [1523393690, 231], [1524127662, 231], [1525256227, 225]], "63fa817636cd56185ae14b0bfd20c979b6dc556e": [[1517092142, 9], [1518360541, 9], [1523393690, 9], [1524127662, 9], [1525256227, 9]], "69d4b80b0959a9cce9f79e147fd678c755f10930": [[1517092142, 3], [1518360541, 3]], "07c0482845cba2cd1e0d7cfc946dc905ad2147e8": [[1517092142, 3]], "5edc20bc8d2c4d9d7fa90ce9d043d9f9be2f88e5": [[1518360541, 7], [1523393690, 7], [1524127662, 7], [1525256227, 7]], "0ef047b5a31c5eb214b97f609c04f5722ccd5e8f": [[1518360541, 3]], "d1012bdb09fb3db72a8d7d8bd5ad61d08d962846": [[1523393690, 18], [1524127662, 18], [1525256227, 18]], "ea65af5c49228edc60e0a045038b049d46da3bfd": [[1523393690, 8], [1524127662, 2], [1525256227, 2]], "a7c9daf6a90f758db31b414ec3a428ef9b86554e": [[1523393690, 1]], "7f346177f8512785f4b9920926b87750f29dff13": [[1523393690, 14], [1524127662, 9], [1525256227, 9]], "57994bc0233d32b05a958eaaf26158c82d642021": [[1523393690, 44], [1524127662, 41], [1525256227, 28]], "afe1bb5877085b9c01cc751b668b31c797a45acd": [[1523393690, 3]], "1f4bfcdeb6dc496390e9bda3174cd5312d591dbd": [[1523393690, 1], [1524127662, 1], [1525256227, 1]], "c06fb2907d9920983bed92e675ffe2d3ab3f8587": [[1524127662, 8], [1525256227, 8]], "aa7a68e4773de6bb9e285e52be8453f98fa36e97": [[1524127662, 56], [1525256227, 56]], "e9774399cf2e15c7cb8dc1643e859787964acced": [[1524127662, 7], [1525256227, 4]], "81c3f9d0ed3189a93343f15d2eb4987d9baad816": [[1524127662, 3], [1525256227, 3]], "c4fe62d060fd5541bf22335beab928ce60e78821": [[1524127662, 2], [1525256227, 2]], "bf3c293c0ab051b8f82a2241708f1ceeb09710d0": [[1524127662, 45]], "060b710bd4faf463abe11d9c8c2916d541a6d51f": [[1524127662, 3]], "ee83910852bda2ac200cd67252c0929c85087427": [[1525256227, 2]], "bd4c1f19e95c87edf6b1d74c3c2a86f82a2e04ce": [[1525256227, 113]], "7bb3e8ca0c0f79d366cbb4dcbe7ab0748fc97080": [[1525256227, 12]], "e7a1463930c1bc486d073ebb4f988e45aea38f8e": [[1525256227, 7]], "f0696db4eae3a966398d43500052767165740da9": [[1525256227, 3]]} -------------------------------------------------------------------------------- /docs/repo/survival.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/miku/esbulk/9984133e8f6c49fb1318686c0ff6a61107baab1c/docs/repo/survival.png -------------------------------------------------------------------------------- /extra/Dockerfile: -------------------------------------------------------------------------------- 1 | ################################ 2 | # STEP 1 build executable binary 3 | ################################ 4 | FROM golang:1.18.2-alpine3.16 AS builder 5 | 6 | # https://github.com/moby/moby/issues/34513#issuecomment-389250632, we hope 7 | # this label is not used for something critical in your setup. 8 | LABEL stage=intermediate 9 | 10 | # Install git, required for fetching the dependencies. 11 | RUN apk update && apk add --no-cache git make 12 | 13 | WORKDIR /app 14 | COPY . . 15 | 16 | # Fetch dependencies, using go get, download only, verbose. 17 | RUN go get -d -v 18 | 19 | # Build the binary. 20 | RUN make esbulk 21 | 22 | ############################ 23 | # STEP 2 build a small image 24 | ############################ 25 | FROM scratch 26 | 27 | # Copy our static executable. 28 | COPY --from=builder /app/esbulk /app/esbulk 29 | 30 | # https://stackoverflow.com/questions/52969195/docker-container-running-golang-http-client-getting-error-certificate-signed-by 31 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 32 | 33 | # Default command. 34 | ENTRYPOINT ["/app/esbulk"] 35 | 36 | -------------------------------------------------------------------------------- /extra/broken.jsonl: -------------------------------------------------------------------------------- 1 | {"a": "hello", "b": 123} 2 | {"a": "world", "b": "234"} 3 | -------------------------------------------------------------------------------- /extra/ok.jsonl: -------------------------------------------------------------------------------- 1 | {"a": "hello", "b": 123} 2 | {"a": "world", "b": 234} 3 | -------------------------------------------------------------------------------- /fixtures/gen.py: -------------------------------------------------------------------------------- 1 | import json 2 | for i in range(10000): 3 | print(json.dumps({"v": "{}".format(i)})) 4 | -------------------------------------------------------------------------------- /flags.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 by Leipzig University Library, http://ub.uni-leipzig.de 2 | // The Finc Authors, http://finc.info 3 | // Martin Czygan, 4 | // 5 | // This file is part of some open source application. 6 | // 7 | // Some open source application is free software: you can redistribute 8 | // it and/or modify it under the terms of the GNU General Public 9 | // License as published by the Free Software Foundation, either 10 | // version 3 of the License, or (at your option) any later version. 11 | // 12 | // Some open source application is distributed in the hope that it will 13 | // be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 14 | // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with Foobar. If not, see . 19 | // 20 | // @license GPL-3.0+ 21 | 22 | package esbulk 23 | 24 | import "strings" 25 | 26 | // ArrayFlags allows to store lists of flag values. 27 | type ArrayFlags []string 28 | 29 | // String representation. 30 | func (f *ArrayFlags) String() string { 31 | return strings.Join(*f, ", ") 32 | } 33 | 34 | // Set appends a value. 35 | func (f *ArrayFlags) Set(value string) error { 36 | *f = append(*f, value) 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/miku/esbulk 2 | 3 | require ( 4 | github.com/klauspost/pgzip v1.2.6 5 | github.com/segmentio/encoding v0.4.0 6 | github.com/sethgrid/pester v1.2.0 7 | github.com/testcontainers/testcontainers-go v0.33.0 8 | ) 9 | 10 | require ( 11 | dario.cat/mergo v1.0.1 // indirect 12 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect 13 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect 14 | github.com/Microsoft/go-winio v0.6.2 // indirect 15 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 16 | github.com/containerd/log v0.1.0 // indirect 17 | github.com/containerd/platforms v0.2.1 // indirect 18 | github.com/cpuguy83/dockercfg v0.3.2 // indirect 19 | github.com/distribution/reference v0.6.0 // indirect 20 | github.com/docker/docker v27.3.1+incompatible // indirect 21 | github.com/docker/go-connections v0.5.0 // indirect 22 | github.com/docker/go-units v0.5.0 // indirect 23 | github.com/felixge/httpsnoop v1.0.4 // indirect 24 | github.com/go-logr/logr v1.4.2 // indirect 25 | github.com/go-logr/stdr v1.2.2 // indirect 26 | github.com/go-ole/go-ole v1.3.0 // indirect 27 | github.com/gogo/protobuf v1.3.2 // indirect 28 | github.com/google/uuid v1.6.0 // indirect 29 | github.com/klauspost/compress v1.17.10 // indirect 30 | github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect 31 | github.com/magiconair/properties v1.8.7 // indirect 32 | github.com/moby/docker-image-spec v1.3.1 // indirect 33 | github.com/moby/patternmatcher v0.6.0 // indirect 34 | github.com/moby/sys/sequential v0.6.0 // indirect 35 | github.com/moby/sys/user v0.3.0 // indirect 36 | github.com/moby/sys/userns v0.1.0 // indirect 37 | github.com/moby/term v0.5.0 // indirect 38 | github.com/morikuni/aec v1.0.0 // indirect 39 | github.com/opencontainers/go-digest v1.0.0 // indirect 40 | github.com/opencontainers/image-spec v1.1.0 // indirect 41 | github.com/pkg/errors v0.9.1 // indirect 42 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect 43 | github.com/segmentio/asm v1.2.0 // indirect 44 | github.com/shirou/gopsutil/v3 v3.24.5 // indirect 45 | github.com/shoenig/go-m1cpu v0.1.6 // indirect 46 | github.com/sirupsen/logrus v1.9.3 // indirect 47 | github.com/tklauser/go-sysconf v0.3.14 // indirect 48 | github.com/tklauser/numcpus v0.9.0 // indirect 49 | github.com/yusufpapurcu/wmi v1.2.4 // indirect 50 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect 51 | go.opentelemetry.io/otel v1.30.0 // indirect 52 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 // indirect 53 | go.opentelemetry.io/otel/metric v1.30.0 // indirect 54 | go.opentelemetry.io/otel/sdk v1.21.0 // indirect 55 | go.opentelemetry.io/otel/trace v1.30.0 // indirect 56 | golang.org/x/crypto v0.28.0 // indirect 57 | golang.org/x/net v0.26.0 // indirect 58 | golang.org/x/sys v0.26.0 // indirect 59 | golang.org/x/time v0.3.0 // indirect 60 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect 61 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect 62 | google.golang.org/protobuf v1.33.0 // indirect 63 | ) 64 | 65 | go 1.22 66 | 67 | toolchain go1.23.1 68 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= 2 | dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= 3 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU= 4 | github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= 5 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= 6 | github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= 7 | github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= 8 | github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= 9 | github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= 10 | github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 11 | github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= 12 | github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= 13 | github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= 14 | github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= 15 | github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= 16 | github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= 17 | github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= 18 | github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= 19 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 21 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 22 | github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= 23 | github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= 24 | github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= 25 | github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= 26 | github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= 27 | github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= 28 | github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= 29 | github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= 30 | github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= 31 | github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= 32 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 33 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 34 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 35 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 36 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 37 | github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= 38 | github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= 39 | github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= 40 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 41 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 42 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 43 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 44 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 45 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 46 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= 47 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= 48 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 49 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 50 | github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= 51 | github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 52 | github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= 53 | github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= 54 | github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0= 55 | github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= 56 | github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= 57 | github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 58 | github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= 59 | github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= 60 | github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= 61 | github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= 62 | github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= 63 | github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= 64 | github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= 65 | github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= 66 | github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= 67 | github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= 68 | github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= 69 | github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= 70 | github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= 71 | github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= 72 | github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= 73 | github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= 74 | github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= 75 | github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= 76 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 77 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 78 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 79 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 80 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= 81 | github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= 82 | github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= 83 | github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= 84 | github.com/segmentio/encoding v0.4.0 h1:MEBYvRqiUB2nfR2criEXWqwdY6HJOUrCn5hboVOVmy8= 85 | github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrWPgtJHYZSnI= 86 | github.com/sethgrid/pester v1.2.0 h1:adC9RS29rRUef3rIKWPOuP1Jm3/MmB6ke+OhE5giENI= 87 | github.com/sethgrid/pester v1.2.0/go.mod h1:hEUINb4RqvDxtoCaU0BNT/HV4ig5kfgOasrf1xcvr0A= 88 | github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= 89 | github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= 90 | github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= 91 | github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= 92 | github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= 93 | github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= 94 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 95 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 96 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 97 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 98 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 99 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 100 | github.com/testcontainers/testcontainers-go v0.33.0 h1:zJS9PfXYT5O0ZFXM2xxXfk4J5UMw/kRiISng037Gxdw= 101 | github.com/testcontainers/testcontainers-go v0.33.0/go.mod h1:W80YpTa8D5C3Yy16icheD01UTDu+LmXIA2Keo+jWtT8= 102 | github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU= 103 | github.com/tklauser/go-sysconf v0.3.14/go.mod h1:1ym4lWMLUOhuBOPGtRcJm7tEGX4SCYNEEEtghGG/8uY= 104 | github.com/tklauser/numcpus v0.9.0 h1:lmyCHtANi8aRUgkckBgoDk1nHCux3n2cgkJLXdQGPDo= 105 | github.com/tklauser/numcpus v0.9.0/go.mod h1:SN6Nq1O3VychhC1npsWostA+oW+VOQTxZrS604NSRyI= 106 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 107 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 108 | github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= 109 | github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= 110 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= 111 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= 112 | go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= 113 | go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= 114 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= 115 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= 116 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= 117 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= 118 | go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= 119 | go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= 120 | go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= 121 | go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= 122 | go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= 123 | go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= 124 | go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= 125 | go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= 126 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 127 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 128 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 129 | golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= 130 | golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= 131 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 132 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 133 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 134 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 135 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 136 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 137 | golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= 138 | golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= 139 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 140 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 141 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 142 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 143 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 144 | golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 145 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 146 | golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 147 | golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 148 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 149 | golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 150 | golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= 151 | golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 152 | golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= 153 | golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= 154 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 155 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 156 | golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= 157 | golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= 158 | golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= 159 | golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 160 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 161 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 162 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 163 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 164 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 165 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 166 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 167 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 168 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= 169 | google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= 170 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= 171 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= 172 | google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= 173 | google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= 174 | google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= 175 | google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= 176 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 177 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 178 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 179 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 180 | gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= 181 | gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= 182 | -------------------------------------------------------------------------------- /indexing.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 by Leipzig University Library, http://ub.uni-leipzig.de 2 | // The Finc Authors, http://finc.info 3 | // Martin Czygan, 4 | // 5 | // This file is part of some open source application. 6 | // 7 | // Some open source application is free software: you can redistribute 8 | // it and/or modify it under the terms of the GNU General Public 9 | // License as published by the Free Software Foundation, either 10 | // version 3 of the License, or (at your option) any later version. 11 | // 12 | // Some open source application is distributed in the hope that it will 13 | // be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 14 | // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with Foobar. If not, see . 19 | // 20 | // @license GPL-3.0+ 21 | 22 | package esbulk 23 | 24 | import ( 25 | "bytes" 26 | "errors" 27 | "fmt" 28 | "io" 29 | "log" 30 | "math/rand" 31 | "net/http" 32 | "strings" 33 | "sync" 34 | "time" 35 | 36 | "github.com/segmentio/encoding/json" 37 | "github.com/sethgrid/pester" 38 | ) 39 | 40 | var errParseCannotServerAddr = errors.New("cannot parse server address") 41 | 42 | // Options represents bulk indexing options. 43 | type Options struct { 44 | Servers []string 45 | Index string 46 | OpType string 47 | DocType string 48 | BatchSize int 49 | Verbose bool 50 | IDField string 51 | Scheme string // http or https; deprecated, use: Servers. 52 | Username string 53 | Password string 54 | Pipeline string 55 | IncludeTypeName bool // https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0 56 | } 57 | 58 | // Item represents a bulk action. 59 | type Item struct { 60 | IndexAction struct { 61 | Index string `json:"_index"` 62 | Type string `json:"_type"` 63 | ID string `json:"_id"` 64 | Status int `json:"status"` 65 | Error struct { 66 | Type string `json:"type"` 67 | Reason string `json:"reason"` 68 | IndexUUID string `json:"index_uuid"` 69 | Shard string `json:"shard"` 70 | Index string `json:"index"` 71 | } `json:"error"` 72 | } `json:"index"` 73 | } 74 | 75 | // BulkResponse is a response to a bulk request. 76 | type BulkResponse struct { 77 | Took int `json:"took"` 78 | HasErrors bool `json:"errors"` 79 | Items []Item `json:"items"` 80 | } 81 | 82 | // nestedStr handles the nested JSON values. 83 | func nestedStr(tokstr []string, docmap map[string]interface{}, currentID string) interface{} { 84 | thistok := tokstr[0] 85 | tempStr2, ok := docmap[thistok].(map[string]interface{}) 86 | if !ok { 87 | return nil 88 | } 89 | var TokenVal interface{} 90 | var ok1 bool 91 | TokenVal = tempStr2 92 | for count3 := 1; count3 < len(tokstr); count3++ { 93 | thistok = tokstr[count3] 94 | TokenVal, ok1 = tempStr2[thistok] 95 | if !ok1 { 96 | return nil 97 | } 98 | if count3 < len(tokstr)-1 { 99 | tempStr2 = TokenVal.(map[string]interface{}) 100 | } 101 | } 102 | return TokenVal 103 | 104 | } 105 | 106 | // BulkIndex takes a set of documents as strings and indexes them into elasticsearch. 107 | func BulkIndex(docs []string, options Options) error { 108 | if len(docs) == 0 { 109 | return nil 110 | } 111 | 112 | rand.Seed(time.Now().Unix()) 113 | server := options.Servers[rand.Intn(len(options.Servers))] 114 | 115 | link := fmt.Sprintf("%s/_bulk", server) 116 | 117 | if options.Pipeline != "" { 118 | link = fmt.Sprintf("%s/_bulk?pipeline=%s", server, options.Pipeline) 119 | } 120 | 121 | var lines []string 122 | for _, doc := range docs { 123 | if len(strings.TrimSpace(doc)) == 0 { 124 | continue 125 | } 126 | var header string 127 | if options.DocType == "" { 128 | header = fmt.Sprintf(`{"%s": {"_index": "%s"}}`, options.OpType, options.Index) 129 | } else { 130 | header = fmt.Sprintf(`{"%s": {"_index": "%s", "_type": "%s"}}`, options.OpType, options.Index, options.DocType) 131 | } 132 | 133 | // If an "-id" is given, peek into the document to extract the ID and 134 | // use it in the header. 135 | if options.IDField != "" { 136 | var docmap map[string]interface{} 137 | dec := json.NewDecoder(strings.NewReader(doc)) 138 | dec.UseNumber() 139 | if err := dec.Decode(&docmap); err != nil { 140 | return fmt.Errorf("failed to json decode doc: %v", err) 141 | } 142 | 143 | idstring := options.IDField // A delimiter separates string with all the fields to be used as ID. 144 | id := strings.FieldsFunc(idstring, func(r rune) bool { return r == ',' || r == ' ' }) 145 | // ID can be any type at this point, try to find a string 146 | // representation or bail out. 147 | var idstr string 148 | var currentID string 149 | for counter := range id { 150 | currentID = id[counter] 151 | tokstr := strings.Split(currentID, ".") 152 | var TokenVal interface{} 153 | if len(tokstr) > 1 { 154 | TokenVal = nestedStr(tokstr, docmap, currentID) 155 | if TokenVal == nil { 156 | return fmt.Errorf("document has no ID field (%s): %s", currentID, doc) 157 | } 158 | } else { 159 | var ok2 bool 160 | TokenVal, ok2 = docmap[currentID] 161 | if !ok2 { 162 | return fmt.Errorf("document has no ID field (%s): %s", currentID, doc) 163 | } 164 | } 165 | switch tempStr1 := interface{}(TokenVal).(type) { 166 | case string: 167 | idstr = idstr + tempStr1 168 | case fmt.Stringer: 169 | idstr = idstr + tempStr1.String() 170 | case json.Number: 171 | idstr = idstr + tempStr1.String() 172 | default: 173 | return fmt.Errorf("cannot convert id value to string") 174 | } 175 | } 176 | 177 | if options.DocType == "" { 178 | header = fmt.Sprintf(`{"%s": {"_index": "%s", "_id": %q}}`, options.OpType, options.Index, idstr) 179 | } else { 180 | header = fmt.Sprintf(`{"%s": {"_index": "%s", "_type": "%s", "_id": %q}}`, 181 | options.OpType, options.Index, options.DocType, idstr) 182 | } 183 | 184 | // Remove the IDField if it is accidentally named '_id', since 185 | // Field [_id] is a metadata field and cannot be added inside a 186 | // document. 187 | var flag int 188 | for count := range id { 189 | if id[count] == "_id" { 190 | flag = 1 // Check if any of the id fields to be concatenated is named '_id'. 191 | } 192 | } 193 | 194 | if flag == 1 { 195 | delete(docmap, "_id") 196 | b, err := json.Marshal(docmap) 197 | if err != nil { 198 | return err 199 | } 200 | doc = string(b) 201 | } 202 | } 203 | 204 | if options.OpType == "update" { 205 | doc = fmt.Sprintf(`{"doc": %s, "doc_as_upsert" : true}`, doc) 206 | } 207 | 208 | lines = append(lines, header, doc) 209 | } 210 | 211 | body := fmt.Sprintf("%s\n", strings.Join(lines, "\n")) 212 | if options.Verbose { 213 | log.Printf("message content-length will be %d", len(body)) 214 | } 215 | 216 | // There are multiple ways indexing can fail, e.g. connection errors or 217 | // bad requests. Finally, if we have a HTTP 200, the bulk request could 218 | // still have failed: for that we need to decode the elasticsearch 219 | // response. 220 | req, err := http.NewRequest("POST", link, strings.NewReader(body)) 221 | if err != nil { 222 | return err 223 | } 224 | 225 | if options.Username != "" && options.Password != "" { 226 | req.SetBasicAuth(options.Username, options.Password) 227 | } 228 | req.Header.Set("Content-Type", "application/json") 229 | response, err := pester.Do(req) 230 | if err != nil { 231 | return err 232 | } 233 | defer response.Body.Close() 234 | 235 | if response.StatusCode >= 400 { 236 | var buf bytes.Buffer 237 | if _, err := io.Copy(&buf, response.Body); err != nil { 238 | return err 239 | } 240 | return fmt.Errorf("indexing failed with %d %s: %s", 241 | response.StatusCode, http.StatusText(response.StatusCode), buf.String()) 242 | } 243 | 244 | var br BulkResponse 245 | if err := json.NewDecoder(response.Body).Decode(&br); err != nil { 246 | return err 247 | } 248 | if br.HasErrors { 249 | if options.Verbose { 250 | log.Println("error details: ") 251 | for _, v := range br.Items { 252 | log.Printf(" %q\n", v.IndexAction.Error) 253 | } 254 | } 255 | log.Printf("request body: %s", body) 256 | return fmt.Errorf("error during bulk operation, check error details; maybe try fewer workers (-w) or increase thread_pool.bulk.queue_size in your nodes") 257 | } 258 | return nil 259 | } 260 | 261 | // Worker will batch index documents that come in on the lines channel. 262 | func Worker(id string, options Options, lines chan string, wg *sync.WaitGroup) { 263 | defer wg.Done() 264 | var docs []string 265 | counter := 0 266 | for s := range lines { 267 | docs = append(docs, s) 268 | counter++ 269 | if counter%options.BatchSize == 0 { 270 | msg := make([]string, len(docs)) 271 | if n := copy(msg, docs); n != len(docs) { 272 | log.Fatalf("expected %d, but got %d", len(docs), n) 273 | } 274 | 275 | if err := BulkIndex(msg, options); err != nil { 276 | log.Fatal(err) 277 | } 278 | if options.Verbose { 279 | log.Printf("[%s] @%d\n", id, counter) 280 | } 281 | docs = nil 282 | } 283 | } 284 | if len(docs) == 0 { 285 | return 286 | } 287 | msg := make([]string, len(docs)) 288 | if n := copy(msg, docs); n != len(docs) { 289 | log.Fatalf("expected %d, but got %d", len(docs), n) 290 | } 291 | 292 | if err := BulkIndex(msg, options); err != nil { 293 | log.Fatal(err) 294 | } 295 | if options.Verbose { 296 | log.Printf("[%s] @%d\n", id, counter) 297 | } 298 | } 299 | 300 | // PutMapping applies a mapping from a reader. 301 | func PutMapping(options Options, body io.Reader) error { 302 | 303 | rand.Seed(time.Now().Unix()) 304 | server := options.Servers[rand.Intn(len(options.Servers))] 305 | var link string 306 | if options.DocType == "" { 307 | link = fmt.Sprintf("%s/%s/_mapping", server, options.Index) 308 | } else { 309 | if options.IncludeTypeName { 310 | // https://www.elastic.co/blog/moving-from-types-to-typeless-apis-in-elasticsearch-7-0 311 | link = fmt.Sprintf("%s/%s/_mapping/%s?include_type_name=true", server, options.Index, options.DocType) 312 | } else { 313 | link = fmt.Sprintf("%s/%s/_mapping/%s", server, options.Index, options.DocType) 314 | } 315 | } 316 | 317 | if options.Verbose { 318 | log.Printf("applying mapping: %s", link) 319 | } 320 | req, err := http.NewRequest("PUT", link, body) 321 | if err != nil { 322 | return err 323 | } 324 | if options.Username != "" && options.Password != "" { 325 | req.SetBasicAuth(options.Username, options.Password) 326 | } 327 | req.Header.Set("Content-Type", "application/json") 328 | resp, err := pester.Do(req) 329 | if err != nil { 330 | return err 331 | } 332 | if resp.StatusCode != 200 { 333 | var buf bytes.Buffer 334 | if _, err := io.Copy(&buf, resp.Body); err != nil { 335 | return err 336 | } 337 | return fmt.Errorf("failed to apply mapping with %s: %s", resp.Status, buf.String()) 338 | } 339 | if options.Verbose { 340 | log.Printf("applied mapping: %s", resp.Status) 341 | } 342 | return resp.Body.Close() 343 | } 344 | 345 | // CreateIndex creates a new index. 346 | func CreateIndex(options Options, body io.Reader) error { 347 | rand.Seed(time.Now().Unix()) 348 | server := options.Servers[rand.Intn(len(options.Servers))] 349 | link := fmt.Sprintf("%s/%s", server, options.Index) 350 | 351 | req, err := http.NewRequest("GET", link, nil) 352 | if err != nil { 353 | return err 354 | } 355 | 356 | if options.Username != "" && options.Password != "" { 357 | req.SetBasicAuth(options.Username, options.Password) 358 | } 359 | req.Header.Set("Content-Type", "application/json") 360 | 361 | resp, err := pester.Do(req) 362 | if err != nil { 363 | return err 364 | } 365 | defer resp.Body.Close() 366 | 367 | // Index already exists, return. 368 | if resp.StatusCode == 200 { 369 | return nil 370 | } 371 | 372 | req, err = http.NewRequest("PUT", fmt.Sprintf("%s/%s/", server, options.Index), body) 373 | 374 | if err != nil { 375 | return err 376 | } 377 | if options.Username != "" && options.Password != "" { 378 | req.SetBasicAuth(options.Username, options.Password) 379 | } 380 | req.Header.Set("Content-Type", "application/json") 381 | resp, err = pester.Do(req) 382 | if err != nil { 383 | return nil 384 | } 385 | defer resp.Body.Close() 386 | 387 | // Elasticsearch backwards compat. 388 | if resp.StatusCode == 400 { 389 | var errResponse struct { 390 | Error string `json:"error"` 391 | Status int `json:"status"` 392 | } 393 | var buf bytes.Buffer 394 | rdr := io.TeeReader(resp.Body, &buf) 395 | // Might return a 400 on "No handler found for uri" ... 396 | if err := json.NewDecoder(rdr).Decode(&errResponse); err == nil { 397 | if strings.Contains(errResponse.Error, "IndexAlreadyExistsException") { 398 | return nil 399 | } 400 | } 401 | log.Printf("elasticsearch response was: %s", buf.String()) 402 | } 403 | if resp.StatusCode >= 400 { 404 | var buf bytes.Buffer 405 | if _, err := io.Copy(&buf, resp.Body); err != nil { 406 | return err 407 | } 408 | return errors.New(buf.String()) 409 | } 410 | if options.Verbose { 411 | log.Printf("created index: %s\n", resp.Status) 412 | } 413 | return nil 414 | } 415 | 416 | // DeleteIndex removes an index. 417 | func DeleteIndex(options Options) error { 418 | rand.Seed(time.Now().Unix()) 419 | server := options.Servers[rand.Intn(len(options.Servers))] 420 | link := fmt.Sprintf("%s/%s", server, options.Index) 421 | 422 | req, err := http.NewRequest("DELETE", link, nil) 423 | if err != nil { 424 | return err 425 | } 426 | if options.Username != "" && options.Password != "" { 427 | req.SetBasicAuth(options.Username, options.Password) 428 | } 429 | req.Header.Set("Content-Type", "application/json") 430 | resp, err := pester.Do(req) 431 | if err != nil { 432 | return err 433 | } 434 | if options.Verbose { 435 | log.Printf("purged index: %s", resp.Status) 436 | } 437 | return resp.Body.Close() 438 | } 439 | -------------------------------------------------------------------------------- /measurements.csv: -------------------------------------------------------------------------------- 1 | es,esbulk,docs,avg_doc_size_b,machines,total_cores,total_heap_gb,duration_s,docs_per_s,replicas 2 | 6.1.2,0.4.8,138000000,2000,1,32,64,6420,22100,1 3 | 6.1.2,0.4.8,138000000,2000,1,8,30,27360,5100,1 4 | 6.1.2,0.4.8,1000000,2000,1,4,1,300,3300,1 5 | 6.1.2,0.4.8,10000000,26,1,4,8,122,81000,1 6 | 6.1.2,0.4.8,10000000,26,1,32,64,32,307000,1 7 | 6.2.3,0.4.10,142944530,2000,2,64,128,26253,5444,1 8 | 6.2.3,0.4.10,142944530,2000,2,64,128,11113,12831,0 9 | 6.2.3,0.4.13,15000000,6000,2,64,128,2460,6400,0 10 | -------------------------------------------------------------------------------- /packaging/debian/esbulk/DEBIAN/control: -------------------------------------------------------------------------------- 1 | Package: esbulk 2 | Version: 0.7.20 3 | Section: utils 4 | Priority: optional 5 | Architecture: amd64 6 | Essential: no 7 | Depends: libc6 (>= 2.12) 8 | Maintainer: Martin Czygan 9 | Description: Fast parallel bulk loading utility for elasticsearch. 10 | -------------------------------------------------------------------------------- /packaging/rpm/buildrpm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | hash rpmbuild 2> /dev/null || { echo >&2 "[EE] rpmbuild executable required"; exit 1; } 4 | 5 | if [ -z "$1" ];then 6 | echo "You didn't specify anything to build"; 7 | exit 1; 8 | fi 9 | 10 | # delete older versions of the rpm since there's no point having old 11 | # versions in there when we still have the src.rpms in the SRPMS dir 12 | find ~/rpmbuild/RPMS -name ${1}-[0-9]\* -exec rm -f {} \; 13 | 14 | cd ~/rpmbuild/SOURCES 15 | 16 | # if there's a directory containing the source, as there will be 17 | # for all our own packages, then delete any .tar.gz file that may exist 18 | # for the package and create a new one. 19 | if [ -d ${1} ] ;then 20 | rm -f ${1}.tar.gz; 21 | tar zcf ${1}.tar.gz ${1}; 22 | fi 23 | 24 | # build the package 25 | rpmbuild -ba ../SPECS/${1}.spec 26 | 27 | # if there is a directory, then delete the .tar.gz again 28 | if [ -d ${1} ] ;then 29 | rm -f ${1}.tar.gz; 30 | fi 31 | -------------------------------------------------------------------------------- /packaging/rpm/esbulk.spec: -------------------------------------------------------------------------------- 1 | Summary: Fast parallel bulk loading utility for elasticsearch. 2 | Name: esbulk 3 | Version: 0.7.20 4 | Release: 0 5 | License: MIT 6 | BuildArch: x86_64 7 | BuildRoot: %{_tmppath}/%{name}-build 8 | Group: System/Base 9 | Vendor: UB Leipzig 10 | URL: https://github.com/miku/esbulk 11 | 12 | %description 13 | 14 | Fast parallel bulk loading utility for elasticsearch. 15 | 16 | %prep 17 | # the set up macro unpacks the source bundle and changes in to the represented by 18 | # %{name} which in this case would be my_maintenance_scripts. So your source bundle 19 | # needs to have a top level directory inside called my_maintenance _scripts 20 | # %setup -n %{name} 21 | 22 | %build 23 | # this section is empty for this example as we're not actually building anything 24 | 25 | %install 26 | # create directories where the files will be located 27 | mkdir -p $RPM_BUILD_ROOT/usr/local/sbin 28 | 29 | # put the files in to the relevant directories. 30 | # the argument on -m is the permissions expressed as octal. (See chmod man page for details.) 31 | install -m 755 esbulk $RPM_BUILD_ROOT/usr/local/sbin 32 | 33 | mkdir -p $RPM_BUILD_ROOT/usr/local/share/man/man1 34 | install -m 644 esbulk.1 $RPM_BUILD_ROOT/usr/local/share/man/man1/esbulk.1 35 | 36 | %post 37 | # the post section is where you can run commands after the rpm is installed. 38 | # insserv /etc/init.d/my_maintenance 39 | 40 | %clean 41 | rm -rf $RPM_BUILD_ROOT 42 | rm -rf %{_tmppath}/%{name} 43 | rm -rf %{_topdir}/BUILD/%{name} 44 | 45 | # list files owned by the package here 46 | %files 47 | %defattr(-,root,root) 48 | /usr/local/sbin/esbulk 49 | /usr/local/share/man/man1/esbulk.1 50 | 51 | %changelog 52 | 53 | * Mon Oct 2 2020 gsocgsoc 54 | - 0.6.2 release 55 | - support for skipping broken json 56 | 57 | * Mon Sep 7 2020 Martin Czygan 58 | - 0.6.1 release 59 | - support for pipelines 60 | - this changelog is infrequetly updated 61 | 62 | * Mon Nov 28 2016 Martin Czygan 63 | - 0.4.2 release 64 | - support for X-Pack, HTTP Basic AUTH, with curl syntax (esbulk -u username:password) 65 | 66 | * Mon Nov 28 2016 Martin Czygan 67 | - 0.4.1 release 68 | - abort indexing, if a single failed bulk request is encountered; do not silently lose documents 69 | 70 | * Sat Nov 26 2016 Martin Czygan 71 | - 0.4.0 release 72 | - attempted fix for https://github.com/miku/esbulk/issues/5 73 | 74 | * Thu Nov 10 2015 Martin Czygan 75 | - 0.3.5 release 76 | - add -mapping and -purge flags 77 | 78 | * Thu May 7 2015 Martin Czygan 79 | - 0.3.3 release 80 | - improve error handling (missing index, wrong index name, ...) 81 | 82 | * Mon Dec 1 2014 Martin Czygan 83 | - 0.3.2 release 84 | - fix panics on connection errors 85 | 86 | * Sun Nov 30 2014 Martin Czygan 87 | - 0.3.1 release 88 | - fix index.refresh_interval settings, make indexing 10-30% faster 89 | 90 | * Sun Nov 30 2014 Martin Czygan 91 | - 0.3 release 92 | - backwards-incompatible changes: removed -q, added -verbose 93 | - added support for gzipped input files 94 | 95 | * Mon Sep 29 2014 Martin Czygan 96 | - 0.2 release, fixed memory leak by closing `response.Body` 97 | 98 | * Tue Aug 26 2014 Martin Czygan 99 | - 0.1 release 100 | -------------------------------------------------------------------------------- /run.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 by Leipzig University Library, http://ub.uni-leipzig.de 2 | // The Finc Authors, http://finc.info 3 | // Martin Czygan, 4 | // 5 | // This file is part of some open source application. 6 | // 7 | // Some open source application is free software: you can redistribute 8 | // it and/or modify it under the terms of the GNU General Public 9 | // License as published by the Free Software Foundation, either 10 | // version 3 of the License, or (at your option) any later version. 11 | // 12 | // Some open source application is distributed in the hope that it will 13 | // be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 14 | // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with Foobar. If not, see . 19 | // 20 | // @license GPL-3.0+ 21 | 22 | package esbulk 23 | 24 | import ( 25 | "bufio" 26 | "errors" 27 | "fmt" 28 | "io" 29 | "log" 30 | "math/rand" 31 | "net/http" 32 | "net/http/httputil" 33 | "os" 34 | "runtime/pprof" 35 | "strings" 36 | "sync" 37 | "time" 38 | 39 | gzip "github.com/klauspost/pgzip" 40 | "github.com/segmentio/encoding/json" 41 | "github.com/sethgrid/pester" 42 | ) 43 | 44 | var ( 45 | // Version of application. 46 | Version = "0.7.20" 47 | 48 | ErrIndexNameRequired = errors.New("index name required") 49 | ErrNoWorkers = errors.New("no workers configured") 50 | ErrInvalidBatchSize = errors.New("cannot use zero batch size") 51 | ) 52 | 53 | // Runner bundles various options. Factored out of a former main func and 54 | // should be further split up (TODO). 55 | type Runner struct { 56 | BatchSize int 57 | Config string 58 | CpuProfile string 59 | OpType string 60 | DocType string 61 | File *os.File 62 | FileGzipped bool 63 | IdentifierField string 64 | IndexName string 65 | Mapping string 66 | MemProfile string 67 | NumWorkers int 68 | Password string 69 | Pipeline string 70 | Purge bool 71 | PurgePause time.Duration 72 | RefreshInterval string 73 | Scheme string 74 | Servers []string 75 | Settings string 76 | ShowVersion bool 77 | SkipBroken bool 78 | Username string 79 | Verbose bool 80 | InsecureSkipVerify bool 81 | ZeroReplica bool 82 | } 83 | 84 | // Run starts indexing documents from file into a given index. 85 | func (r *Runner) Run() (err error) { 86 | if r.ShowVersion { 87 | fmt.Println(Version) 88 | return nil 89 | } 90 | if r.NumWorkers == 0 { 91 | return ErrNoWorkers 92 | } 93 | if r.BatchSize == 0 { 94 | return ErrInvalidBatchSize 95 | } 96 | if r.CpuProfile != "" { 97 | f, err := os.Create(r.CpuProfile) 98 | if err != nil { 99 | return err 100 | } 101 | pprof.StartCPUProfile(f) 102 | defer pprof.StopCPUProfile() 103 | } 104 | if r.IndexName == "" { 105 | return ErrIndexNameRequired 106 | } 107 | if r.OpType == "" { 108 | r.OpType = "index" 109 | } 110 | if len(r.Servers) == 0 { 111 | r.Servers = append(r.Servers, "http://localhost:9200") 112 | } 113 | r.Servers = mapString(prependSchema, r.Servers) 114 | if r.Verbose { 115 | log.Printf("using %d server(s)", len(r.Servers)) 116 | } 117 | options := Options{ 118 | Servers: r.Servers, 119 | Index: r.IndexName, 120 | OpType: r.OpType, 121 | DocType: r.DocType, 122 | BatchSize: r.BatchSize, 123 | Verbose: r.Verbose, 124 | Scheme: "http", // deprecated 125 | IDField: r.IdentifierField, 126 | Username: r.Username, 127 | Password: r.Password, 128 | Pipeline: r.Pipeline, 129 | } 130 | if r.Verbose { 131 | log.Println(options) 132 | } 133 | if r.Purge { 134 | if err := DeleteIndex(options); err != nil { 135 | return err 136 | } 137 | time.Sleep(r.PurgePause) 138 | } 139 | var createIndexBody io.Reader 140 | if r.Config != "" { 141 | if _, err := os.Stat(r.Config); os.IsNotExist(err) { 142 | createIndexBody = strings.NewReader(r.Config) 143 | } else { 144 | file, err := os.Open(r.Config) 145 | if err != nil { 146 | return err 147 | } 148 | defer file.Close() 149 | createIndexBody = bufio.NewReader(file) 150 | } 151 | } 152 | if err := CreateIndex(options, createIndexBody); err != nil { 153 | return err 154 | } 155 | if r.Mapping != "" { 156 | var reader io.Reader 157 | if _, err := os.Stat(r.Mapping); os.IsNotExist(err) { 158 | reader = strings.NewReader(r.Mapping) 159 | } else { 160 | file, err := os.Open(r.Mapping) 161 | if err != nil { 162 | return err 163 | } 164 | defer file.Close() 165 | reader = bufio.NewReader(file) 166 | } 167 | err := PutMapping(options, reader) 168 | if err != nil { 169 | return err 170 | } 171 | } 172 | var ( 173 | queue = make(chan string) 174 | wg sync.WaitGroup 175 | ) 176 | wg.Add(r.NumWorkers) 177 | for i := 0; i < r.NumWorkers; i++ { 178 | name := fmt.Sprintf("worker-%d", i) 179 | go Worker(name, options, queue, &wg) 180 | } 181 | if r.Verbose { 182 | log.Printf("started %d workers", r.NumWorkers) 183 | } 184 | for i, _ := range options.Servers { 185 | // Store number_of_replicas settings for restoration later. 186 | doc, err := GetSettings(i, options) 187 | if err != nil { 188 | return err 189 | } 190 | // TODO(miku): Rework this. 191 | numberOfReplicas := doc[options.Index].(map[string]interface{})["settings"].(map[string]interface{})["index"].(map[string]interface{})["number_of_replicas"] 192 | if r.Verbose { 193 | log.Printf("on shutdown, number_of_replicas will be set back to %s", numberOfReplicas) 194 | } 195 | if r.Verbose { 196 | log.Printf("on shutdown, refresh_interval will be set back to %s", r.RefreshInterval) 197 | } 198 | // Shutdown procedure. TODO(miku): Handle signals, too. 199 | defer func() { 200 | // Realtime search. 201 | if _, err = indexSettingsRequest(fmt.Sprintf(`{"index": {"refresh_interval": "%s"}}`, r.RefreshInterval), options); err != nil { 202 | return 203 | } 204 | // Reset number of replicas. 205 | if _, err = indexSettingsRequest(fmt.Sprintf(`{"index": {"number_of_replicas": %q}}`, numberOfReplicas), options); err != nil { 206 | return 207 | } 208 | // Persist documents. 209 | err = FlushIndex(i, options) 210 | }() 211 | // Realtime search. 212 | resp, err := indexSettingsRequest(`{"index": {"refresh_interval": "-1"}}`, options) 213 | if err != nil { 214 | return err 215 | } 216 | if resp.StatusCode >= 400 { 217 | b, err := httputil.DumpResponse(resp, true) 218 | if err != nil { 219 | return err 220 | } 221 | return fmt.Errorf("got %v: %v", resp.StatusCode, string(b)) 222 | } 223 | if r.ZeroReplica { 224 | // Reset number of replicas. 225 | if _, err := indexSettingsRequest(`{"index": {"number_of_replicas": 0}}`, options); err != nil { 226 | return err 227 | } 228 | } 229 | } 230 | var ( 231 | reader = bufio.NewReader(r.File) 232 | counter = 0 233 | start = time.Now() 234 | ) 235 | if r.FileGzipped { 236 | zreader, err := gzip.NewReader(r.File) 237 | if err != nil { 238 | log.Fatal(err) 239 | } 240 | reader = bufio.NewReader(zreader) 241 | } 242 | if r.Verbose && r.File != nil { 243 | log.Printf("start reading from %v", r.File.Name()) 244 | } 245 | for { 246 | line, err := reader.ReadString('\n') 247 | if err == io.EOF { 248 | break 249 | } 250 | if err != nil { 251 | return err 252 | } 253 | if line = strings.TrimSpace(line); len(line) == 0 { 254 | continue 255 | } 256 | if r.SkipBroken { 257 | if !(isJSON(line)) { 258 | if r.Verbose { 259 | fmt.Printf("skipped line [%s]\n", line) 260 | } 261 | continue 262 | } 263 | } 264 | queue <- line 265 | counter++ 266 | } 267 | close(queue) 268 | wg.Wait() 269 | elapsed := time.Since(start) 270 | if r.MemProfile != "" { 271 | f, err := os.Create(r.MemProfile) 272 | if err != nil { 273 | return err 274 | } 275 | pprof.WriteHeapProfile(f) 276 | f.Close() 277 | } 278 | if r.Verbose { 279 | elapsed := elapsed.Seconds() 280 | if elapsed < 0.1 { 281 | elapsed = 0.1 282 | } 283 | rate := float64(counter) / elapsed 284 | log.Printf("%d docs in %0.2fs at %0.3f docs/s with %d workers\n", counter, elapsed, rate, r.NumWorkers) 285 | } 286 | return nil 287 | } 288 | 289 | // indexSettingsRequest runs updates an index setting, given a body and 290 | // options. Body consist of the JSON document, e.g. `{"index": 291 | // {"refresh_interval": "1s"}}`. 292 | func indexSettingsRequest(body string, options Options) (*http.Response, error) { 293 | r := strings.NewReader(body) 294 | 295 | rand.Seed(time.Now().Unix()) 296 | server := options.Servers[rand.Intn(len(options.Servers))] 297 | link := fmt.Sprintf("%s/%s/_settings", server, options.Index) 298 | 299 | req, err := http.NewRequest("PUT", link, r) 300 | if err != nil { 301 | return nil, err 302 | } 303 | // Auth handling. 304 | if options.Username != "" && options.Password != "" { 305 | req.SetBasicAuth(options.Username, options.Password) 306 | } 307 | req.Header.Set("Content-Type", "application/json") 308 | 309 | resp, err := pester.Do(req) 310 | if err != nil { 311 | return nil, err 312 | } 313 | if options.Verbose { 314 | log.Printf("applied setting: %s with status %s\n", body, resp.Status) 315 | } 316 | return resp, nil 317 | } 318 | 319 | // isJSON checks if a string is valid json. 320 | func isJSON(str string) bool { 321 | var js json.RawMessage 322 | return json.Unmarshal([]byte(str), &js) == nil 323 | } 324 | 325 | func prependSchema(s string) string { 326 | if !strings.HasPrefix(s, "http") { 327 | return fmt.Sprintf("http://%s", s) 328 | } 329 | return s 330 | } 331 | 332 | func mapString(f func(string) string, vs []string) (result []string) { 333 | for _, v := range vs { 334 | result = append(result, f(v)) 335 | } 336 | return 337 | } 338 | -------------------------------------------------------------------------------- /run_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2021 by Leipzig University Library, http://ub.uni-leipzig.de 2 | // The Finc Authors, http://finc.info 3 | // Martin Czygan, 4 | // 5 | // This file is part of some open source application. 6 | // 7 | // Some open source application is free software: you can redistribute 8 | // it and/or modify it under the terms of the GNU General Public 9 | // License as published by the Free Software Foundation, either 10 | // version 3 of the License, or (at your option) any later version. 11 | // 12 | // Some open source application is distributed in the hope that it will 13 | // be useful, but WITHOUT ANY WARRANTY; without even the implied warranty 14 | // of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | // GNU General Public License for more details. 16 | // 17 | // You should have received a copy of the GNU General Public License 18 | // along with Foobar. If not, see . 19 | // 20 | // @license GPL-3.0+ 21 | 22 | package esbulk 23 | 24 | import ( 25 | "context" 26 | "fmt" 27 | "io" 28 | "io/ioutil" 29 | "log" 30 | "net/url" 31 | "os" 32 | "os/exec" 33 | "os/user" 34 | "strings" 35 | "testing" 36 | "time" 37 | 38 | "github.com/segmentio/encoding/json" 39 | "github.com/sethgrid/pester" 40 | "github.com/testcontainers/testcontainers-go" 41 | "github.com/testcontainers/testcontainers-go/wait" 42 | ) 43 | 44 | func TestIncompleteConfig(t *testing.T) { 45 | skipNoDocker(t) 46 | var cases = []struct { 47 | help string 48 | r Runner 49 | err error 50 | }{ 51 | {help: "no default index name", r: Runner{}, err: ErrNoWorkers}, 52 | {help: "no such host", r: Runner{ 53 | IndexName: "abc", 54 | BatchSize: 10, 55 | NumWorkers: 1, 56 | Servers: []string{"http://broken.server:9200"}, 57 | }, err: &url.Error{Op: "Get", URL: "http://broken.server:9200/abc"}}, 58 | } 59 | for _, c := range cases { 60 | err := c.r.Run() 61 | switch err.(type) { 62 | case nil: 63 | if c.err != nil { 64 | t.Fatalf("got: %#v, %T, want: %v [%s]", err, err, c.err, c.help) 65 | } 66 | case *url.Error: 67 | // For now, only check whether we expect an error. 68 | if c.err == nil { 69 | t.Fatalf("got: %#v, %T, want: %v [%s]", err, err, c.err, c.help) 70 | } 71 | default: 72 | if err != c.err { 73 | t.Fatalf("got: %#v, %T, want: %v [%s]", err, err, c.err, c.help) 74 | } 75 | } 76 | } 77 | } 78 | 79 | // startServer starts an elasticsearch server from image, exposing the http 80 | // port. Note that the Java heap required may be 2GB or more. 81 | func startServer(ctx context.Context, image string, httpPort int) (testcontainers.Container, error) { 82 | var ( 83 | hp = fmt.Sprintf("%d:9200/tcp", httpPort) 84 | parts = strings.Split(image, ":") 85 | tag string 86 | ) 87 | if len(parts) == 2 { 88 | tag = parts[1] 89 | } else { 90 | tag = "latest" 91 | } 92 | var ( 93 | name = fmt.Sprintf("esbulk-test-es-%s-%d", tag, time.Now().UnixNano()) 94 | req = testcontainers.ContainerRequest{ 95 | Image: image, 96 | Name: name, 97 | Env: map[string]string{ 98 | "discovery.type": "single-node", 99 | // If you’re starting a single-node Elasticsearch cluster in a 100 | // Docker container, security will be automatically enabled and 101 | // configured for you. -- https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html#docker-cli-run-dev-mode 102 | "xpack.security.enabled": "false", 103 | "ES_JAVA_OPTS": "-Xms4g -Xmx4g", 104 | }, 105 | ExposedPorts: []string{hp}, 106 | WaitingFor: wait.ForLog("started"), 107 | } 108 | ) 109 | return testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ 110 | ProviderType: testcontainers.ProviderPodman, 111 | ContainerRequest: req, 112 | Started: true, 113 | }) 114 | } 115 | 116 | // logReader reads data from reader and bot logs it and returns it. Fails, if 117 | // reading fails. 118 | func logReader(t *testing.T, r io.Reader) []byte { 119 | b, err := ioutil.ReadAll(r) 120 | if err != nil { 121 | t.Fatalf("read failed: %s", err) 122 | return nil 123 | } 124 | t.Logf("%s", string(b)) 125 | return b 126 | } 127 | 128 | // skipNoDocker skips a test, if docker is not running. Also support podman. 129 | func skipNoDocker(t *testing.T) { 130 | noDocker := false 131 | cmd := exec.Command("systemctl", "is-active", "docker") 132 | b, err := cmd.CombinedOutput() 133 | if err != nil { 134 | noDocker = true 135 | } 136 | if strings.TrimSpace(string(b)) != "active" { 137 | noDocker = true 138 | } 139 | if !noDocker { 140 | // We found some docker. 141 | return 142 | } 143 | // Otherwise, try podman. 144 | _, err = exec.LookPath("podman") 145 | if err == nil { 146 | t.Logf("podman detected") 147 | // DOCKER_HOST=unix:///run/user/$UID/podman/podman.sock 148 | usr, err := user.Current() 149 | if err != nil { 150 | t.Logf("cannot get UID, set DOCKER_HOST manually") 151 | } else { 152 | sckt := fmt.Sprintf("unix:///run/user/%v/podman/podman.sock", usr.Uid) 153 | os.Setenv("DOCKER_HOST", sckt) 154 | t.Logf("set DOCKER_HOST to %v", sckt) 155 | } 156 | noDocker = false 157 | } 158 | if noDocker { 159 | t.Skipf("docker not installed or not running") 160 | } 161 | } 162 | 163 | func TestMinimalConfig(t *testing.T) { 164 | skipNoDocker(t) 165 | ctx := context.Background() 166 | var imageConf = []struct { 167 | ElasticsearchMajorVersion int 168 | Image string 169 | HttpPort int 170 | }{ 171 | {2, "elasticsearch:2.3.4", 39200}, 172 | {5, "elasticsearch:5.6.16", 39200}, 173 | {6, "elasticsearch:6.8.14", 39200}, 174 | {7, "elasticsearch:7.17.0", 39200}, // https://is.gd/MPwhaM, https://is.gd/RJ4LOZ, ... 175 | {8, "elasticsearch:8.6.0", 39200}, 176 | } 177 | log.Printf("testing %d versions: %v", len(imageConf), imageConf) 178 | for _, conf := range imageConf { 179 | wrapper := func() error { 180 | c, err := startServer(ctx, conf.Image, conf.HttpPort) 181 | if err != nil { 182 | t.Fatalf("could not start test container for %v: %v", conf.Image, err) 183 | } 184 | defer func() { 185 | if err := c.Terminate(ctx); err != nil { 186 | t.Errorf("could not kill container: %v", err) 187 | } 188 | }() 189 | base := fmt.Sprintf("http://localhost:%d", conf.HttpPort) 190 | resp, err := pester.Get(base) 191 | if err != nil { 192 | t.Fatalf("request failed: %v", err) 193 | } 194 | defer resp.Body.Close() 195 | logReader(t, resp.Body) 196 | t.Logf("server should be up at %s", base) 197 | 198 | var cases = []struct { 199 | filename string 200 | indexName string 201 | numDocs int64 202 | err error 203 | }{ 204 | {"fixtures/v10k.jsonl", "abc", 10000, nil}, 205 | } 206 | for _, c := range cases { 207 | f, err := os.Open(c.filename) 208 | if err != nil { 209 | return fmt.Errorf("could not open fixture: %s", c.filename) 210 | } 211 | defer f.Close() 212 | r := Runner{ 213 | Servers: []string{"http://localhost:39200"}, 214 | BatchSize: 5000, 215 | NumWorkers: 1, 216 | RefreshInterval: "1s", 217 | IndexName: "abc", 218 | File: f, 219 | Verbose: true, 220 | } 221 | if conf.ElasticsearchMajorVersion < 7 { 222 | r.DocType = "any" // deprecated with ES7, fails with ES8 223 | } 224 | err = r.Run() 225 | if err != c.err { 226 | return fmt.Errorf("got %v, want %v", err, c.err) 227 | } 228 | searchURL := fmt.Sprintf("%s/%s/_search", base, r.IndexName) 229 | resp, err = pester.Get(searchURL) 230 | if err != nil { 231 | return fmt.Errorf("could not query es: %v", err) 232 | } 233 | defer resp.Body.Close() 234 | b := logReader(t, resp.Body) 235 | var ( 236 | sr7 SearchResponse7 237 | sr6 SearchResponse6 238 | ) 239 | if err = json.Unmarshal(b, &sr7); err != nil { 240 | if err = json.Unmarshal(b, &sr6); err != nil { 241 | t.Errorf("could not parse json response (6, 7): %v", err) 242 | } else { 243 | t.Log("es6 detected") 244 | } 245 | } 246 | if sr7.Hits.Total.Value != c.numDocs && sr6.Hits.Total != c.numDocs { 247 | t.Errorf("expected %d docs", c.numDocs) 248 | } 249 | } 250 | // Save logs. 251 | rc, err := c.Logs(ctx) 252 | if err != nil { 253 | log.Printf("logs not available: %v", err) 254 | } 255 | if err := os.MkdirAll("logs", 0755); err != nil { 256 | if !os.IsExist(err) { 257 | log.Printf("create dir failed: %v", err) 258 | } 259 | } 260 | cname, err := c.Name(ctx) 261 | if err != nil { 262 | t.Logf("failed to get container name: %v", err) 263 | } 264 | fn := fmt.Sprintf("logs/%s-%s.log", time.Now().Format("20060102150405"), strings.TrimLeft(cname, "/")) 265 | f, err := os.Create(fn) 266 | if err != nil { 267 | log.Printf("failed to create log file: %v", err) 268 | } 269 | defer f.Close() 270 | log.Printf("logging to %s", fn) 271 | if _, err := io.Copy(f, rc); err != nil { 272 | log.Printf("log failed: %v", err) 273 | } 274 | return nil 275 | } 276 | if err := wrapper(); err != nil { 277 | t.Fatal(err) 278 | } 279 | } 280 | } 281 | 282 | func TestGH32(t *testing.T) { 283 | skipNoDocker(t) 284 | ctx := context.Background() 285 | c, err := startServer(ctx, "elasticsearch:7.17.0", 39200) 286 | if err != nil { 287 | t.Fatalf("could not start test container: %v", err) 288 | } 289 | defer func() { 290 | if err := c.Terminate(ctx); err != nil { 291 | t.Errorf("could not kill container: %v", err) 292 | } 293 | }() 294 | base := fmt.Sprintf("http://localhost:%d", 39200) 295 | resp, err := pester.Get(base) 296 | if err != nil { 297 | t.Fatalf("request failed: %v", err) 298 | } 299 | defer resp.Body.Close() 300 | logReader(t, resp.Body) 301 | t.Logf("server should be up at %s", base) 302 | 303 | f, err := os.Open("fixtures/v10k.jsonl") 304 | if err != nil { 305 | t.Errorf("could not open fixture: %v", err) 306 | } 307 | defer f.Close() 308 | r := Runner{ 309 | Servers: []string{"http://localhost:39200"}, 310 | BatchSize: 5000, 311 | NumWorkers: 1, 312 | RefreshInterval: "1s", 313 | Mapping: `{}`, 314 | IndexName: "abc", 315 | DocType: "any", // deprecated with ES7 316 | File: f, 317 | Verbose: true, 318 | } 319 | // this should fail with #32 320 | err = r.Run() 321 | if err != nil { 322 | t.Logf("expected err: %v", err) 323 | } else { 324 | t.Fatalf("expected fail, see #32") 325 | } 326 | // w/o doctype, we should be good 327 | r = Runner{ 328 | Servers: []string{"http://localhost:39200"}, 329 | BatchSize: 5000, 330 | NumWorkers: 1, 331 | RefreshInterval: "1s", 332 | Mapping: `{}`, 333 | IndexName: "abc", 334 | File: f, 335 | Verbose: true, 336 | } 337 | err = r.Run() 338 | if err != nil { 339 | t.Fatalf("unexpected failure: %v", err) 340 | } 341 | } 342 | 343 | type SearchResponse6 struct { 344 | Hits struct { 345 | Hits []struct { 346 | Id string `json:"_id"` 347 | Index string `json:"_index"` 348 | Score float64 `json:"_score"` 349 | Source struct { 350 | V string `json:"v"` 351 | } `json:"_source"` 352 | Type string `json:"_type"` 353 | } `json:"hits"` 354 | MaxScore float64 `json:"max_score"` 355 | Total int64 `json:"total"` 356 | } `json:"hits"` 357 | Shards struct { 358 | Failed int64 `json:"failed"` 359 | Skipped int64 `json:"skipped"` 360 | Successful int64 `json:"successful"` 361 | Total int64 `json:"total"` 362 | } `json:"_shards"` 363 | TimedOut bool `json:"timed_out"` 364 | Took int64 `json:"took"` 365 | } 366 | 367 | type SearchResponse7 struct { 368 | Hits struct { 369 | Hits []struct { 370 | Id string `json:"_id"` 371 | Index string `json:"_index"` 372 | Score float64 `json:"_score"` 373 | Source struct { 374 | V string `json:"v"` 375 | } `json:"_source"` 376 | Type string `json:"_type"` 377 | } `json:"hits"` 378 | MaxScore float64 `json:"max_score"` 379 | Total struct { 380 | Relation string `json:"relation"` 381 | Value int64 `json:"value"` 382 | } `json:"total"` 383 | } `json:"hits"` 384 | Shards struct { 385 | Failed int64 `json:"failed"` 386 | Skipped int64 `json:"skipped"` 387 | Successful int64 `json:"successful"` 388 | Total int64 `json:"total"` 389 | } `json:"_shards"` 390 | TimedOut bool `json:"timed_out"` 391 | Took int64 `json:"took"` 392 | } 393 | --------------------------------------------------------------------------------