├── .gitignore ├── LICENSE ├── README.md ├── example ├── bench-2020-11-07-1.txt ├── bench-2020-11-07-2.txt └── bench_test.go ├── go.mod ├── init ├── systemd │ ├── README.md │ └── bench.service └── upstart │ ├── README.md │ └── bench.conf ├── install.sh ├── internal ├── benchfmt │ └── fmt.go ├── cpupower │ └── cpupower.go ├── lock │ ├── client.go │ ├── cred_windows.go │ ├── creds_darwin.go │ ├── creds_linux.go │ ├── daemon_darwin.go │ ├── daemon_linux.go │ ├── daemon_windows.go │ ├── flag.go │ ├── lock.go │ └── proto.go ├── stat │ ├── algo.go │ ├── beta.go │ ├── data.go │ ├── delta.go │ ├── mathx.go │ ├── normaldist.go │ ├── sample.go │ ├── scaler.go │ ├── sort.go │ ├── table.go │ ├── tdist.go │ ├── text.go │ ├── ttest.go │ ├── udist.go │ └── utest.go └── term │ └── color.go └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bench 2 | 3 | Reliable performance measurement for Go programs. All in one design. 4 | 5 | ``` 6 | $ go install golang.design/x/bench@latest 7 | ``` 8 | 9 | ## Features 10 | 11 | - Combine [benchstat](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat), [perflock](https://github.com/aclements/perflock) and more... 12 | - Short command and only run benchmarks 13 | - Automatic performance locking for benchmarks 14 | - Automatic statistic analysis for benchmark results 15 | - Color indications for benchmark results 16 | 17 | ## Usage 18 | 19 | ### Enable `bench` Daemon (optional, Linux only) 20 | 21 | ```sh 22 | $ cd $GOPATH/src/golang.design/x/bench 23 | $ ./install.bash 24 | ``` 25 | 26 | If your init system is supported, this will also configure `bench` to start automatically on boot. 27 | 28 | Or you can install and run `bench` daemon manually: 29 | 30 | ```sh 31 | $ sudo install $GOPATH/bin/bench /usr/bin/bench 32 | $ sudo -b bench -daemon 33 | ``` 34 | 35 | ### Default Behavior 36 | 37 | ```sh 38 | $ bench 39 | ``` 40 | 41 | The detault behavior of `bench` run benchmarks under your current 42 | working directory, and each benchmark will be ran 10 times for further 43 | statistical analysis. It will also try to acquire performance lock from 44 | `bench` daemon to gain more stable results. Furthermore, the benchmark 45 | results are saved as a text file to the working directory and named as 46 | `.txt`. 47 | 48 | Example: 49 | 50 | ``` 51 | $ cd example 52 | $ bench 53 | bench: run benchmarks under 90% cpufreq... 54 | bench: go test -run=^$ -bench=. -count=10 55 | goos: linux 56 | goarch: amd64 57 | pkg: golang.design/x/bench/example 58 | BenchmarkDemo-16 21114 57340 ns/op 59 | ... 60 | BenchmarkDemo-16 21004 57097 ns/op 61 | PASS 62 | ok golang.design/x/bench/example 17.791s 63 | bench: results are saved to file: ./bench-2020-11-07-19:59:51.txt 64 | 65 | name time/op 66 | Demo-16 57.0µs ±1% 67 | 68 | $ # ... do some changes to the benchmark ... 69 | 70 | $ bench 71 | bench: run benchmarks under 90% cpufreq... 72 | bench: go test -run=^$ -bench=. -count=10 73 | goos: linux 74 | goarch: amd64 75 | pkg: golang.design/x/bench/example 76 | BenchmarkDemo-16 213145 5625 ns/op 77 | ... 78 | BenchmarkDemo-16 212959 5632 ns/op 79 | PASS 80 | ok golang.design/x/bench/example 12.536s 81 | bench: results are saved to file: ./bench-2020-11-07-20:00:16.txt 82 | 83 | name time/op 84 | Demo-16 5.63µs ±0% 85 | 86 | $ bench bench-2020-11-07-19:59:51.txt bench-2020-11-07-20:00:16.txt 87 | name old time/op new time/op delta 88 | Demo-16 57.0µs ±1% 5.6µs ±0% -90.13% (p=0.000 n=10+8) 89 | ``` 90 | 91 | ### Options 92 | 93 | Options for checking daemon status: 94 | 95 | ```sh 96 | bench -list 97 | ``` 98 | 99 | Options for statistic tests: 100 | 101 | ```sh 102 | bench old.txt [new.txt] # same from benchstat 103 | bench -delta-test 104 | bench -alpha 105 | bench -geomean 106 | bench -split 107 | bench -sort 108 | ``` 109 | 110 | Options for running benchmarks: 111 | 112 | ```sh 113 | bench -v # enable verbose outputs 114 | bench -shared # enable shared execution 115 | bench -cpufreq 90 # cpu frequency (default: 90) 116 | bench -name BenchmarkXXX # go test `-bench` flag (default: .) 117 | bench -count 20 # go test `-count` flag (default: 10) 118 | bench -time 100x # go test `-benchtime` flag (default: unset) 119 | bench -cpuproc 1,2,4,8,16,32,128 # go test `-cpu` flag (default: unset) 120 | ``` 121 | 122 | ## License 123 | 124 | © 2020 The golang.design Authors -------------------------------------------------------------------------------- /example/bench-2020-11-07-1.txt: -------------------------------------------------------------------------------- 1 | goos: linux 2 | goarch: amd64 3 | pkg: golang.design/x/bench/example 4 | BenchmarkDemo-16 21114 57340 ns/op 5 | BenchmarkDemo-16 21109 56897 ns/op 6 | BenchmarkDemo-16 21091 56833 ns/op 7 | BenchmarkDemo-16 21121 56821 ns/op 8 | BenchmarkDemo-16 21105 56953 ns/op 9 | BenchmarkDemo-16 21112 56830 ns/op 10 | BenchmarkDemo-16 20836 57499 ns/op 11 | BenchmarkDemo-16 21084 56885 ns/op 12 | BenchmarkDemo-16 20990 57245 ns/op 13 | BenchmarkDemo-16 21004 57097 ns/op 14 | PASS 15 | ok golang.design/x/bench/example 17.791s 16 | -------------------------------------------------------------------------------- /example/bench-2020-11-07-2.txt: -------------------------------------------------------------------------------- 1 | goos: linux 2 | goarch: amd64 3 | pkg: golang.design/x/bench/example 4 | BenchmarkDemo-16 213145 5625 ns/op 5 | BenchmarkDemo-16 211988 5629 ns/op 6 | BenchmarkDemo-16 211838 5629 ns/op 7 | BenchmarkDemo-16 212151 5627 ns/op 8 | BenchmarkDemo-16 212577 5627 ns/op 9 | BenchmarkDemo-16 210150 5635 ns/op 10 | BenchmarkDemo-16 212650 5637 ns/op 11 | BenchmarkDemo-16 210063 5670 ns/op 12 | BenchmarkDemo-16 212568 5665 ns/op 13 | BenchmarkDemo-16 212959 5632 ns/op 14 | PASS 15 | ok golang.design/x/bench/example 12.536s 16 | -------------------------------------------------------------------------------- /example/bench_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The golang.design Initiative Authors. 2 | // All rights reserved. Use of this source code is governed 3 | // by a GNU GPLv3 license that can be found in the LICENSE file. 4 | 5 | package example 6 | 7 | import "testing" 8 | 9 | func grow(n int) { 10 | if n > 0 { 11 | grow(n - 1) 12 | } 13 | } 14 | 15 | func BenchmarkDemo(b *testing.B) { 16 | for i := 0; i < b.N; i++ { 17 | grow(10000) // try change this and re-run `bench` 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module golang.design/x/bench 2 | 3 | go 1.15 4 | -------------------------------------------------------------------------------- /init/systemd/README.md: -------------------------------------------------------------------------------- 1 | To configure systemd to run `bench`, run 2 | 3 | ``` 4 | $ sudo install -m 0644 bench.service /etc/systemd/system 5 | $ sudo systemctl enable --now bench.service 6 | ``` -------------------------------------------------------------------------------- /init/systemd/bench.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Bench daemon 3 | 4 | [Service] 5 | Type=simple 6 | ExecStart=/usr/bin/bench -daemon 7 | Restart=on-failure 8 | 9 | [Install] 10 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /init/upstart/README.md: -------------------------------------------------------------------------------- 1 | To configure Upstart to run `bench`, copy `bench.conf` into `/etc/init/` and run sudo start `bench`. E.g., 2 | 3 | ```sh 4 | $ sudo install -m 0644 bench.conf /etc/init/ 5 | $ sudo start bench 6 | ``` -------------------------------------------------------------------------------- /init/upstart/bench.conf: -------------------------------------------------------------------------------- 1 | # bench - Benchmark performance locking service 2 | 3 | description "bench daemon" 4 | 5 | start on runlevel [2345] 6 | stop on runlevel [!2345] 7 | respawn 8 | 9 | console log 10 | 11 | exec bench -daemon -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | BINPATH="$(go env GOBIN)" 6 | if [[ -z "$BINPATH" ]]; then 7 | BINPATH="$(go env GOPATH)/bin" 8 | fi 9 | BIN="$BINPATH/bench" 10 | if [[ ! -x "$BIN" ]]; then 11 | echo "bench binary $BIN does not exist." 2>&1 12 | echo "Please run go install golang.design/x/bench" 2>&1 13 | exit 1 14 | fi 15 | 16 | echo "Installing $BIN to /usr/bin" 1>&2 17 | sudo install "$BIN" /usr/bin/bench 18 | 19 | start="-b /usr/bin/bench -daemon" 20 | starttype= 21 | if [[ -d /etc/init ]]; then 22 | echo "Installing init script for Upstart" 1>&2 23 | sudo install -m 0644 init/upstart/bench.conf /etc/init/ 24 | start="service bench start" 25 | starttype=" (using Upstart)" 26 | fi 27 | if [[ -d /etc/systemd ]]; then 28 | echo "Installing service for systemd" 1>&2 29 | sudo install -m 0644 init/systemd/bench.service /etc/systemd/system 30 | sudo systemctl enable --quiet bench.service 31 | start="systemctl start bench.service" 32 | starttype=" (using systemd)" 33 | fi 34 | 35 | if /usr/bin/bench -list >/dev/null 2>&1; then 36 | echo "Not starting bench daemon (already running)" 1>&2 37 | else 38 | echo "Starting bench daemon$starttype" 1>&2 39 | sudo $start 40 | fi -------------------------------------------------------------------------------- /internal/benchfmt/fmt.go: -------------------------------------------------------------------------------- 1 | // Package benchfmt provides readers and writers for the Go benchmark format. 2 | // 3 | // The format is documented at https://golang.org/design/14313-benchmark-format 4 | package benchfmt 5 | 6 | import ( 7 | "bufio" 8 | "bytes" 9 | "fmt" 10 | "io" 11 | "sort" 12 | "strconv" 13 | "strings" 14 | "unicode" 15 | ) 16 | 17 | // Reader reads benchmark results from an io.Reader. 18 | // Use Next to advance through the results. 19 | // 20 | // br := benchfmt.NewReader(r) 21 | // for br.Next() { 22 | // res := br.Result() 23 | // ... 24 | // } 25 | // err = br.Err() // get any error encountered during iteration 26 | // ... 27 | type Reader struct { 28 | s *bufio.Scanner 29 | labels Labels 30 | // permLabels are permanent labels read from the start of the 31 | // file or provided by AddLabels. They cannot be overridden. 32 | permLabels Labels 33 | lineNum int 34 | // cached from last call to newResult, to save on allocations 35 | lastName string 36 | lastNameLabels Labels 37 | // cached from the last call to Next 38 | result *Result 39 | err error 40 | } 41 | 42 | // NewReader creates a BenchmarkReader that reads from r. 43 | func NewReader(r io.Reader) *Reader { 44 | return &Reader{ 45 | s: bufio.NewScanner(r), 46 | labels: make(Labels), 47 | } 48 | } 49 | 50 | // AddLabels adds additional labels as if they had been read from the header of a file. 51 | // It must be called before the first call to r.Next. 52 | func (r *Reader) AddLabels(labels Labels) { 53 | r.permLabels = labels.Copy() 54 | for k, v := range labels { 55 | r.labels[k] = v 56 | } 57 | } 58 | 59 | // Result represents a single line from a benchmark file. 60 | // All information about that line is self-contained in the Result. 61 | // A Result is immutable once created. 62 | type Result struct { 63 | // Labels is the set of persistent labels that apply to the result. 64 | // Labels must not be modified. 65 | Labels Labels 66 | // NameLabels is the set of ephemeral labels that were parsed 67 | // from the benchmark name/line. 68 | // NameLabels must not be modified. 69 | NameLabels Labels 70 | // LineNum is the line number on which the result was found 71 | LineNum int 72 | // Content is the verbatim input line of the benchmark file, beginning with the string "Benchmark". 73 | Content string 74 | } 75 | 76 | // SameLabels reports whether r and b have the same labels. 77 | func (r *Result) SameLabels(b *Result) bool { 78 | return r.Labels.Equal(b.Labels) && r.NameLabels.Equal(b.NameLabels) 79 | } 80 | 81 | // Labels is a set of key-value strings. 82 | type Labels map[string]string 83 | 84 | // String returns the labels formatted as a comma-separated 85 | // list enclosed in braces. 86 | func (l Labels) String() string { 87 | var out bytes.Buffer 88 | out.WriteString("{") 89 | for k, v := range l { 90 | fmt.Fprintf(&out, "%q: %q, ", k, v) 91 | } 92 | if out.Len() > 1 { 93 | // Remove extra ", " 94 | out.Truncate(out.Len() - 2) 95 | } 96 | out.WriteString("}") 97 | return out.String() 98 | } 99 | 100 | // Keys returns a sorted list of the keys in l. 101 | func (l Labels) Keys() []string { 102 | var out []string 103 | for k := range l { 104 | out = append(out, k) 105 | } 106 | sort.Strings(out) 107 | return out 108 | } 109 | 110 | // Equal reports whether l and b have the same keys and values. 111 | func (l Labels) Equal(b Labels) bool { 112 | if len(l) != len(b) { 113 | return false 114 | } 115 | for k := range l { 116 | if l[k] != b[k] { 117 | return false 118 | } 119 | } 120 | return true 121 | } 122 | 123 | // A Printer prints a sequence of benchmark results. 124 | type Printer struct { 125 | w io.Writer 126 | labels Labels 127 | } 128 | 129 | // NewPrinter constructs a BenchmarkPrinter writing to w. 130 | func NewPrinter(w io.Writer) *Printer { 131 | return &Printer{w: w} 132 | } 133 | 134 | // Print writes the lines necessary to recreate r. 135 | func (p *Printer) Print(r *Result) error { 136 | var keys []string 137 | // Print removed keys first. 138 | for k := range p.labels { 139 | if r.Labels[k] == "" { 140 | keys = append(keys, k) 141 | } 142 | } 143 | sort.Strings(keys) 144 | for _, k := range keys { 145 | if _, err := fmt.Fprintf(p.w, "%s:\n", k); err != nil { 146 | return err 147 | } 148 | } 149 | // Then print new or changed keys. 150 | keys = keys[:0] 151 | for k, v := range r.Labels { 152 | if v != "" && p.labels[k] != v { 153 | keys = append(keys, k) 154 | } 155 | } 156 | sort.Strings(keys) 157 | for _, k := range keys { 158 | if _, err := fmt.Fprintf(p.w, "%s: %s\n", k, r.Labels[k]); err != nil { 159 | return err 160 | } 161 | } 162 | // Finally print the actual line itself. 163 | if _, err := fmt.Fprintf(p.w, "%s\n", r.Content); err != nil { 164 | return err 165 | } 166 | p.labels = r.Labels 167 | return nil 168 | } 169 | 170 | // parseNameLabels extracts extra labels from a benchmark name and sets them in labels. 171 | func parseNameLabels(name string, labels Labels) { 172 | dash := strings.LastIndex(name, "-") 173 | if dash >= 0 { 174 | // Accept -N as an alias for /gomaxprocs=N 175 | _, err := strconv.Atoi(name[dash+1:]) 176 | if err == nil { 177 | labels["gomaxprocs"] = name[dash+1:] 178 | name = name[:dash] 179 | } 180 | } 181 | parts := strings.Split(name, "/") 182 | labels["name"] = parts[0] 183 | for i, sub := range parts[1:] { 184 | equals := strings.Index(sub, "=") 185 | var key string 186 | if equals >= 0 { 187 | key, sub = sub[:equals], sub[equals+1:] 188 | } else { 189 | key = fmt.Sprintf("sub%d", i+1) 190 | } 191 | labels[key] = sub 192 | } 193 | } 194 | 195 | // newResult parses a line and returns a Result object for the line. 196 | func (r *Reader) newResult(labels Labels, lineNum int, name, content string) *Result { 197 | res := &Result{ 198 | Labels: labels, 199 | LineNum: lineNum, 200 | Content: content, 201 | } 202 | if r.lastName != name { 203 | r.lastName = name 204 | r.lastNameLabels = make(Labels) 205 | parseNameLabels(name, r.lastNameLabels) 206 | } 207 | res.NameLabels = r.lastNameLabels 208 | return res 209 | } 210 | 211 | // Copy returns a new copy of the labels map, to protect against 212 | // future modifications to labels. 213 | func (l Labels) Copy() Labels { 214 | new := make(Labels) 215 | for k, v := range l { 216 | new[k] = v 217 | } 218 | return new 219 | } 220 | 221 | // TODO(quentin): How to represent and efficiently group multiple lines? 222 | 223 | // Next returns the next benchmark result from the file. If there are 224 | // no further results, it returns nil, io.EOF. 225 | func (r *Reader) Next() bool { 226 | if r.err != nil { 227 | return false 228 | } 229 | copied := false 230 | havePerm := r.permLabels != nil 231 | for r.s.Scan() { 232 | r.lineNum++ 233 | line := r.s.Text() 234 | if key, value, ok := parseKeyValueLine(line); ok { 235 | if _, ok := r.permLabels[key]; ok { 236 | continue 237 | } 238 | if !copied { 239 | copied = true 240 | r.labels = r.labels.Copy() 241 | } 242 | // TODO(quentin): Spec says empty value is valid, but 243 | // we need a way to cancel previous labels, so we'll 244 | // treat an empty value as a removal. 245 | if value == "" { 246 | delete(r.labels, key) 247 | } else { 248 | r.labels[key] = value 249 | } 250 | continue 251 | } 252 | // Blank line delimits the header. If we find anything else, the file must not have a header. 253 | if !havePerm { 254 | if line == "" { 255 | r.permLabels = r.labels.Copy() 256 | } else { 257 | r.permLabels = Labels{} 258 | } 259 | } 260 | if fullName, ok := parseBenchmarkLine(line); ok { 261 | r.result = r.newResult(r.labels, r.lineNum, fullName, line) 262 | return true 263 | } 264 | } 265 | if err := r.s.Err(); err != nil { 266 | r.err = err 267 | return false 268 | } 269 | r.err = io.EOF 270 | return false 271 | } 272 | 273 | // Result returns the most recent result generated by a call to Next. 274 | func (r *Reader) Result() *Result { 275 | return r.result 276 | } 277 | 278 | // Err returns the error state of the reader. 279 | func (r *Reader) Err() error { 280 | if r.err == io.EOF { 281 | return nil 282 | } 283 | return r.err 284 | } 285 | 286 | // parseKeyValueLine attempts to parse line as a key: value pair. ok 287 | // indicates whether the line could be parsed. 288 | func parseKeyValueLine(line string) (key, val string, ok bool) { 289 | for i, c := range line { 290 | if i == 0 && !unicode.IsLower(c) { 291 | return 292 | } 293 | if unicode.IsSpace(c) || unicode.IsUpper(c) { 294 | return 295 | } 296 | if i > 0 && c == ':' { 297 | key = line[:i] 298 | val = line[i+1:] 299 | break 300 | } 301 | } 302 | if key == "" { 303 | return 304 | } 305 | if val == "" { 306 | ok = true 307 | return 308 | } 309 | for len(val) > 0 && (val[0] == ' ' || val[0] == '\t') { 310 | val = val[1:] 311 | ok = true 312 | } 313 | return 314 | } 315 | 316 | // parseBenchmarkLine attempts to parse line as a benchmark result. If 317 | // successful, fullName is the name of the benchmark with the 318 | // "Benchmark" prefix stripped, and ok is true. 319 | func parseBenchmarkLine(line string) (fullName string, ok bool) { 320 | space := strings.IndexFunc(line, unicode.IsSpace) 321 | if space < 0 { 322 | return 323 | } 324 | name := line[:space] 325 | if !strings.HasPrefix(name, "Benchmark") { 326 | return 327 | } 328 | return name[len("Benchmark"):], true 329 | } 330 | -------------------------------------------------------------------------------- /internal/cpupower/cpupower.go: -------------------------------------------------------------------------------- 1 | // Package cpupower manipulates Linux CPU frequency scaling settings. 2 | package cpupower 3 | 4 | import ( 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "regexp" 10 | "sort" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | // Domain is a frequency scaling domain. This may include more than 16 | // one CPU. 17 | type Domain struct { 18 | path string 19 | min, max int 20 | available []int 21 | } 22 | 23 | var cpuRe = regexp.MustCompile(`cpu\d+$`) 24 | 25 | // Domains returns the frequency scaling domains of this host. 26 | func Domains() ([]*Domain, error) { 27 | dir := "/sys/devices/system/cpu" 28 | fs, err := ioutil.ReadDir(dir) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | var domains []*Domain 34 | haveDomains := make(map[string]bool) 35 | for _, f := range fs { 36 | if !f.IsDir() || !cpuRe.MatchString(f.Name()) { 37 | continue 38 | } 39 | pdir := filepath.Join(dir, f.Name(), "cpufreq") 40 | 41 | // Get the frequency domain, if any. 42 | cpus, err := ioutil.ReadFile(filepath.Join(pdir, "freqdomain_cpus")) 43 | if err == nil { 44 | if haveDomains[string(cpus)] { 45 | // We already have a CPU in this domain. 46 | continue 47 | } 48 | haveDomains[string(cpus)] = true 49 | } else if !os.IsNotExist(err) { 50 | return nil, err 51 | } 52 | 53 | min, err := readInt(filepath.Join(pdir, "cpuinfo_min_freq")) 54 | if err != nil { 55 | return nil, err 56 | } 57 | max, err := readInt(filepath.Join(pdir, "cpuinfo_max_freq")) 58 | if err != nil { 59 | return nil, err 60 | } 61 | avail, err := readInts(filepath.Join(pdir, "scaling_available_frequencies")) 62 | if err != nil && !os.IsNotExist(err) { 63 | return nil, err 64 | } 65 | sort.Ints(avail) 66 | domains = append(domains, &Domain{pdir, min, max, avail}) 67 | } 68 | return domains, nil 69 | } 70 | 71 | // AvailableRange returns the available frequency range this CPU is 72 | // capable of and the set of available frequencies in ascending order 73 | // or nil if any frequency can be set. 74 | func (d *Domain) AvailableRange() (int, int, []int) { 75 | return d.min, d.max, d.available 76 | } 77 | 78 | // CurrentRange returns the current frequency range this CPU's 79 | // governor can select between. 80 | func (d *Domain) CurrentRange() (int, int, error) { 81 | min, err := readInt(filepath.Join(d.path, "scaling_min_freq")) 82 | if err != nil { 83 | return 0, 0, err 84 | } 85 | max, err := readInt(filepath.Join(d.path, "scaling_max_freq")) 86 | if err != nil { 87 | return 0, 0, err 88 | } 89 | return min, max, nil 90 | } 91 | 92 | // SetRange sets the frequency range this CPU's governor can select 93 | // between. 94 | func (d *Domain) SetRange(min, max int) error { 95 | // Attempting to set an empty range will cause an IO error. 96 | // Rather than trying to figure out the right order to set 97 | // them in, try both orders. 98 | err1 := writeInt(filepath.Join(d.path, "scaling_min_freq"), min) 99 | if err2 := writeInt(filepath.Join(d.path, "scaling_max_freq"), max); err2 != nil { 100 | return err2 101 | } 102 | if err1 != nil { 103 | err1 = writeInt(filepath.Join(d.path, "scaling_min_freq"), min) 104 | } 105 | return err1 106 | } 107 | 108 | func readInt(path string) (int, error) { 109 | data, err := ioutil.ReadFile(path) 110 | if err != nil { 111 | return 0, err 112 | } 113 | return strconv.Atoi(strings.TrimSpace(string(data))) 114 | } 115 | 116 | func writeInt(path string, val int) error { 117 | return ioutil.WriteFile(path, []byte(fmt.Sprintf("%d", val)), 0) 118 | } 119 | 120 | func readInts(path string) ([]int, error) { 121 | data, err := ioutil.ReadFile(path) 122 | if err != nil { 123 | return nil, err 124 | } 125 | fields := strings.Fields(string(data)) 126 | ints := make([]int, len(fields)) 127 | for i, field := range fields { 128 | ints[i], err = strconv.Atoi(field) 129 | if err != nil { 130 | return nil, err 131 | } 132 | } 133 | return ints, nil 134 | } 135 | -------------------------------------------------------------------------------- /internal/lock/client.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import ( 4 | "encoding/gob" 5 | "fmt" 6 | "log" 7 | "net" 8 | ) 9 | 10 | // Client is a lock client 11 | type Client struct { 12 | c net.Conn 13 | 14 | gr *gob.Encoder 15 | gw *gob.Decoder 16 | } 17 | 18 | // NewClient returns a lock client 19 | func NewClient() *Client { 20 | c, err := net.Dial("unix", Socketpath) 21 | if err != nil { 22 | log.Printf("failed to connect bench daemon: %v", err) 23 | return nil 24 | } 25 | 26 | // Send credentials. 27 | err = writeCredentials(c.(*net.UnixConn)) 28 | if err != nil { 29 | log.Fatal("failed to send credentials: ", err) 30 | } 31 | 32 | gr, gw := gob.NewEncoder(c), gob.NewDecoder(c) 33 | 34 | return &Client{c, gr, gw} 35 | } 36 | 37 | func (c *Client) do(action perflockAction, response interface{}) { 38 | err := c.gr.Encode(action) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | err = c.gw.Decode(response) 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | } 48 | 49 | // Acquire acuiqres the lock 50 | func (c *Client) Acquire(shared, nonblocking bool, msg string) bool { 51 | var ok bool 52 | c.do(perflockAction{actionAcquire{Shared: shared, NonBlocking: nonblocking, Msg: msg}}, &ok) 53 | return ok 54 | } 55 | 56 | // List lists all perflock actions 57 | func (c *Client) List() []string { 58 | var list []string 59 | c.do(perflockAction{actionList{}}, &list) 60 | return list 61 | } 62 | 63 | // SetCPUFreq sets the given cpu frequency 64 | func (c *Client) SetCPUFreq(percent int) error { 65 | var err string 66 | c.do(perflockAction{actionSetCPUFreq{Percent: percent}}, &err) 67 | if err == "" { 68 | return nil 69 | } 70 | return fmt.Errorf("%s", err) 71 | } 72 | -------------------------------------------------------------------------------- /internal/lock/cred_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package lock 5 | 6 | import ( 7 | "errors" 8 | "net" 9 | ) 10 | 11 | func writeCredentials(c *net.UnixConn) error { 12 | return errors.New("unimplemented") 13 | } 14 | -------------------------------------------------------------------------------- /internal/lock/creds_darwin.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | // +build darwin 3 | 4 | package lock 5 | 6 | import ( 7 | "errors" 8 | "net" 9 | ) 10 | 11 | func writeCredentials(c *net.UnixConn) error { 12 | return errors.New("unimplemented") 13 | } 14 | -------------------------------------------------------------------------------- /internal/lock/creds_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package lock 5 | 6 | import ( 7 | "fmt" 8 | "net" 9 | "os" 10 | "syscall" 11 | ) 12 | 13 | // TODO: Use SO_PEERCRED instead? 14 | 15 | func writeCredentials(c *net.UnixConn) error { 16 | ucred := syscall.Ucred{Pid: int32(os.Getpid()), Uid: uint32(os.Getuid()), Gid: uint32(os.Getgid())} 17 | credOob := syscall.UnixCredentials(&ucred) 18 | credMsg := []byte("x") 19 | n, oobn, err := c.WriteMsgUnix(credMsg, credOob, nil) 20 | if err != nil { 21 | return err 22 | } 23 | if n != 1 { 24 | return fmt.Errorf("short send (%d bytes)", n) 25 | } 26 | if oobn != len(credOob) { 27 | return fmt.Errorf("short OOB send (%d bytes)", oobn) 28 | } 29 | return nil 30 | } 31 | 32 | func readCredentials(c *net.UnixConn) (*syscall.Ucred, error) { 33 | // Enable receiving credentials on c. 34 | f, err := c.File() 35 | if err != nil { 36 | return nil, err 37 | } 38 | err = syscall.SetsockoptInt(int(f.Fd()), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1) 39 | f.Close() 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | // Receive credentials. 45 | buf := make([]byte, 1) 46 | oob := make([]byte, 128) 47 | n, oobn, _, _, err := c.ReadMsgUnix(buf, oob) 48 | if err != nil { 49 | return nil, err 50 | } 51 | if n != 1 { 52 | return nil, fmt.Errorf("expected 1 byte, got %d", n) 53 | } 54 | 55 | // Parse OOB data. 56 | scms, err := syscall.ParseSocketControlMessage(oob[:oobn]) 57 | if err != nil { 58 | return nil, err 59 | } 60 | if len(scms) != 1 { 61 | return nil, fmt.Errorf("expected 1 control message, got %d", len(scms)) 62 | } 63 | return syscall.ParseUnixCredentials(&scms[0]) 64 | } 65 | -------------------------------------------------------------------------------- /internal/lock/daemon_darwin.go: -------------------------------------------------------------------------------- 1 | //go:build darwin 2 | // +build darwin 3 | 4 | package lock 5 | 6 | import ( 7 | "log" 8 | ) 9 | 10 | // RunDaemon runs lock daemon 11 | func RunDaemon() { 12 | log.Fatal("running daemon on darwin systems are not supported.") 13 | } 14 | -------------------------------------------------------------------------------- /internal/lock/daemon_linux.go: -------------------------------------------------------------------------------- 1 | //go:build linux 2 | // +build linux 3 | 4 | package lock 5 | 6 | import ( 7 | "encoding/gob" 8 | "fmt" 9 | "io" 10 | "log" 11 | "net" 12 | "os" 13 | "os/user" 14 | "time" 15 | 16 | "golang.design/x/bench/internal/cpupower" 17 | ) 18 | 19 | var theLock perflock 20 | 21 | // RunDaemon runs lock daemon 22 | func RunDaemon() { 23 | // check if daemon is running 24 | c, _ := net.Dial("unix", Socketpath) 25 | if c != nil { 26 | c.Close() 27 | log.Fatalf("The bench daemon is already running at %s !", Socketpath) 28 | return 29 | } 30 | 31 | os.Remove(Socketpath) 32 | l, err := net.Listen("unix", Socketpath) 33 | if err != nil { 34 | log.Fatal(err) 35 | } 36 | defer l.Close() 37 | 38 | // Make the socket world-writable/connectable. 39 | err = os.Chmod(Socketpath, 0777) 40 | if err != nil { 41 | log.Fatal(err) 42 | } 43 | 44 | // Receive connections. 45 | for { 46 | conn, err := l.Accept() 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | 51 | go func(c net.Conn) { 52 | defer c.Close() 53 | NewServer(c).Serve() 54 | }(conn) 55 | } 56 | } 57 | 58 | // Server is the bench lock server 59 | type Server struct { 60 | c net.Conn 61 | userName string 62 | 63 | locker *locker 64 | acquiring bool 65 | 66 | oldCPUFreqs []*cpuFreqSettings 67 | } 68 | 69 | // NewServer returns a bench lock server 70 | func NewServer(c net.Conn) *Server { 71 | return &Server{c: c} 72 | } 73 | 74 | // Serve serves the bench lock server 75 | func (s *Server) Serve() { 76 | // Drop any held locks if we exit for any reason. 77 | defer s.drop() 78 | 79 | // Get connection credentials. 80 | ucred, err := readCredentials(s.c.(*net.UnixConn)) 81 | if err != nil { 82 | log.Print("reading credentials: ", err) 83 | return 84 | } 85 | 86 | u, err := user.LookupId(fmt.Sprintf("%d", ucred.Uid)) 87 | s.userName = "???" 88 | if err == nil { 89 | s.userName = u.Username 90 | } 91 | 92 | // Receive incoming actions. We do this in a goroutine so the 93 | // main handler can select on EOF or lock acquisition. 94 | actions := make(chan perflockAction) 95 | go func() { 96 | gr := gob.NewDecoder(s.c) 97 | for { 98 | var msg perflockAction 99 | err := gr.Decode(&msg) 100 | if err != nil { 101 | if err != io.EOF { 102 | log.Print(err) 103 | } 104 | close(actions) 105 | return 106 | } 107 | actions <- msg 108 | } 109 | }() 110 | 111 | // Process incoming actions. 112 | var acquireC <-chan bool 113 | gw := gob.NewEncoder(s.c) 114 | for { 115 | select { 116 | case action, ok := <-actions: 117 | if !ok { 118 | // Connection closed. 119 | return 120 | } 121 | if s.acquiring { 122 | log.Printf("protocol error: message while acquiring") 123 | return 124 | } 125 | switch action := action.Action.(type) { 126 | case actionAcquire: 127 | if s.locker != nil { 128 | log.Printf("protocol error: acquiring lock twice") 129 | return 130 | } 131 | msg := fmt.Sprintf("%s\t%s\t%s", s.userName, time.Now().Format(time.Stamp), action.Msg) 132 | if action.Shared { 133 | msg += " [shared]" 134 | } 135 | s.locker = theLock.Enqueue(action.Shared, action.NonBlocking, msg) 136 | if s.locker != nil { 137 | // Enqueued. Wait for acquire. 138 | s.acquiring = true 139 | acquireC = s.locker.C 140 | } else { 141 | // Non-blocking acquire failed. 142 | if err := gw.Encode(false); err != nil { 143 | log.Print(err) 144 | return 145 | } 146 | } 147 | 148 | case actionList: 149 | list := theLock.Queue() 150 | if err := gw.Encode(list); err != nil { 151 | log.Print(err) 152 | return 153 | } 154 | 155 | case actionSetCPUFreq: 156 | if s.locker == nil { 157 | log.Printf("protocol error: setting cpuFreq without lock") 158 | return 159 | } 160 | err := s.setCPUFreq(action.Percent) 161 | errString := "" 162 | if err != nil { 163 | errString = err.Error() 164 | } 165 | if err := gw.Encode(errString); err != nil { 166 | log.Print(err) 167 | return 168 | } 169 | 170 | default: 171 | log.Printf("unknown message") 172 | return 173 | } 174 | 175 | case <-acquireC: 176 | // Lock acquired. 177 | s.acquiring, acquireC = false, nil 178 | if err := gw.Encode(true); err != nil { 179 | log.Print(err) 180 | return 181 | } 182 | } 183 | } 184 | } 185 | 186 | func (s *Server) drop() { 187 | // Restore the CPU cpuFreq before releasing the lock. 188 | if s.oldCPUFreqs != nil { 189 | s.restoreCPUFreq() 190 | s.oldCPUFreqs = nil 191 | } 192 | // Release the lock. 193 | if s.locker != nil { 194 | theLock.Dequeue(s.locker) 195 | s.locker = nil 196 | } 197 | } 198 | 199 | type cpuFreqSettings struct { 200 | domain *cpupower.Domain 201 | min, max int 202 | } 203 | 204 | func (s *Server) setCPUFreq(percent int) error { 205 | domains, err := cpupower.Domains() 206 | if err != nil { 207 | return err 208 | } 209 | if len(domains) == 0 { 210 | return fmt.Errorf("no power domains") 211 | } 212 | 213 | // Save current frequency settings. 214 | old := []*cpuFreqSettings{} 215 | for _, d := range domains { 216 | min, max, err := d.CurrentRange() 217 | if err != nil { 218 | return err 219 | } 220 | old = append(old, &cpuFreqSettings{d, min, max}) 221 | } 222 | s.oldCPUFreqs = old 223 | 224 | // Set new settings. 225 | abs := func(x int) int { 226 | if x < 0 { 227 | return -x 228 | } 229 | return x 230 | } 231 | for _, d := range domains { 232 | min, max, avail := d.AvailableRange() 233 | target := (max-min)*percent/100 + min 234 | 235 | // Find the nearest available frequency. 236 | if len(avail) != 0 { 237 | closest := avail[0] 238 | for _, a := range avail { 239 | if abs(target-a) < abs(target-closest) { 240 | closest = a 241 | } 242 | } 243 | target = closest 244 | } 245 | 246 | err := d.SetRange(target, target) 247 | if err != nil { 248 | return err 249 | } 250 | } 251 | 252 | return nil 253 | } 254 | 255 | func (s *Server) restoreCPUFreq() error { 256 | var err error 257 | for _, g := range s.oldCPUFreqs { 258 | // Try to set all of the domains, even if one fails. 259 | err1 := g.domain.SetRange(g.min, g.max) 260 | if err1 != nil && err == nil { 261 | err = err1 262 | } 263 | } 264 | return err 265 | } 266 | -------------------------------------------------------------------------------- /internal/lock/daemon_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package lock 5 | 6 | import ( 7 | "log" 8 | ) 9 | 10 | // RunDaemon runs lock daemon 11 | func RunDaemon() { 12 | log.Fatal("running daemon on windows systems are not supported.") 13 | } 14 | -------------------------------------------------------------------------------- /internal/lock/flag.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strconv" 7 | ) 8 | 9 | // Socketpath between lock daemon and client 10 | var Socketpath = "/var/run/bench.socket" 11 | 12 | // CpufreqFlag ... 13 | type CpufreqFlag struct { 14 | Percent int 15 | } 16 | 17 | func (f *CpufreqFlag) String() string { 18 | if f.Percent < 0 { 19 | return "none" 20 | } 21 | return fmt.Sprintf("%d", f.Percent) 22 | } 23 | 24 | // Set set the cpu frequency percentage 25 | func (f *CpufreqFlag) Set(v string) error { 26 | if v == "none" { 27 | f.Percent = -1 28 | } else { 29 | m := regexp.MustCompile(`^([0-9]+)$`).FindStringSubmatch(v) 30 | if m == nil { 31 | return fmt.Errorf("cpufreq must be \"none\" or \"N\"") 32 | } 33 | f.Percent, _ = strconv.Atoi(m[1]) 34 | } 35 | return nil 36 | } 37 | -------------------------------------------------------------------------------- /internal/lock/lock.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import "sync" 4 | 5 | type perflock struct { 6 | l sync.Mutex 7 | q []*locker 8 | } 9 | 10 | type locker struct { 11 | C <-chan bool 12 | c chan<- bool 13 | shared bool 14 | woken bool 15 | 16 | msg string 17 | } 18 | 19 | func (l *perflock) Enqueue(shared, nonblocking bool, msg string) *locker { 20 | ch := make(chan bool, 1) 21 | locker := &locker{ch, ch, shared, false, msg} 22 | 23 | // Enqueue. 24 | l.l.Lock() 25 | defer l.l.Unlock() 26 | l.setQ(append(l.q, locker)) 27 | 28 | if nonblocking && !locker.woken { 29 | // Acquire failed. Dequeue. 30 | l.setQ(l.q[:len(l.q)-1]) 31 | return nil 32 | } 33 | 34 | return locker 35 | } 36 | 37 | func (l *perflock) Dequeue(locker *locker) { 38 | l.l.Lock() 39 | defer l.l.Unlock() 40 | for i, o := range l.q { 41 | if locker == o { 42 | copy(l.q[i:], l.q[i+1:]) 43 | l.setQ(l.q[:len(l.q)-1]) 44 | return 45 | } 46 | } 47 | panic("Dequeue of non-enqueued locker") 48 | } 49 | 50 | func (l *perflock) Queue() []string { 51 | var q []string 52 | 53 | l.l.Lock() 54 | defer l.l.Unlock() 55 | for _, locker := range l.q { 56 | q = append(q, locker.msg) 57 | } 58 | return q 59 | } 60 | 61 | func (l *perflock) setQ(q []*locker) { 62 | l.q = q 63 | if len(q) == 0 { 64 | return 65 | } 66 | 67 | wake := func(locker *locker) { 68 | if locker.woken == false { 69 | locker.woken = true 70 | locker.c <- true 71 | } 72 | } 73 | if q[0].shared { 74 | // Wake all shared acquires at the head of the queue. 75 | for _, locker := range q { 76 | if !locker.shared { 77 | break 78 | } 79 | wake(locker) 80 | } 81 | } else { 82 | wake(q[0]) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /internal/lock/proto.go: -------------------------------------------------------------------------------- 1 | package lock 2 | 3 | import "encoding/gob" 4 | 5 | type perflockAction struct { 6 | Action interface{} 7 | } 8 | 9 | // actionAcquire acquires the lock. The response is a boolean 10 | // indicating whether or not the lock was acquired (which may be false 11 | // for a non-blocking acquire). 12 | type actionAcquire struct { 13 | Shared bool 14 | NonBlocking bool 15 | Msg string 16 | } 17 | 18 | // actionList returns the list of current and pending lock 19 | // acquisitions as a []string. 20 | type actionList struct { 21 | } 22 | 23 | // actionSetCPUFreq sets the CPU frequency of all CPUs. The caller 24 | // must hold the lock. 25 | type actionSetCPUFreq struct { 26 | // Percent indicates the percent to set the CPU cpuFreq to 27 | // between the lower and highest available frequencies. 28 | Percent int 29 | } 30 | 31 | func init() { 32 | gob.Register(actionAcquire{}) 33 | gob.Register(actionList{}) 34 | gob.Register(actionSetCPUFreq{}) 35 | } 36 | -------------------------------------------------------------------------------- /internal/stat/algo.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | // Miscellaneous helper algorithms 4 | 5 | import ( 6 | "fmt" 7 | ) 8 | 9 | func maxint(a, b int) int { 10 | if a > b { 11 | return a 12 | } 13 | return b 14 | } 15 | 16 | func minint(a, b int) int { 17 | if a < b { 18 | return a 19 | } 20 | return b 21 | } 22 | 23 | func sumint(xs []int) int { 24 | sum := 0 25 | for _, x := range xs { 26 | sum += x 27 | } 28 | return sum 29 | } 30 | 31 | // bisect returns an x in [low, high] such that |f(x)| <= tolerance 32 | // using the bisection method. 33 | // 34 | // f(low) and f(high) must have opposite signs. 35 | // 36 | // If f does not have a root in this interval (e.g., it is 37 | // discontiguous), this returns the X of the apparent discontinuity 38 | // and false. 39 | func bisect(f func(float64) float64, low, high, tolerance float64) (float64, bool) { 40 | flow, fhigh := f(low), f(high) 41 | if -tolerance <= flow && flow <= tolerance { 42 | return low, true 43 | } 44 | if -tolerance <= fhigh && fhigh <= tolerance { 45 | return high, true 46 | } 47 | if mathSign(flow) == mathSign(fhigh) { 48 | panic(fmt.Sprintf("root of f is not bracketed by [low, high]; f(%g)=%g f(%g)=%g", low, flow, high, fhigh)) 49 | } 50 | for { 51 | mid := (high + low) / 2 52 | fmid := f(mid) 53 | if -tolerance <= fmid && fmid <= tolerance { 54 | return mid, true 55 | } 56 | if mid == high || mid == low { 57 | return mid, false 58 | } 59 | if mathSign(fmid) == mathSign(flow) { 60 | low = mid 61 | flow = fmid 62 | } else { 63 | high = mid 64 | fhigh = fmid 65 | } 66 | } 67 | } 68 | 69 | // bisectBool implements the bisection method on a boolean function. 70 | // It returns x1, x2 ∈ [low, high], x1 < x2 such that f(x1) != f(x2) 71 | // and x2 - x1 <= xtol. 72 | // 73 | // If f(low) == f(high), it panics. 74 | func bisectBool(f func(float64) bool, low, high, xtol float64) (x1, x2 float64) { 75 | flow, fhigh := f(low), f(high) 76 | if flow == fhigh { 77 | panic(fmt.Sprintf("root of f is not bracketed by [low, high]; f(%g)=%v f(%g)=%v", low, flow, high, fhigh)) 78 | } 79 | for { 80 | if high-low <= xtol { 81 | return low, high 82 | } 83 | mid := (high + low) / 2 84 | if mid == high || mid == low { 85 | return low, high 86 | } 87 | fmid := f(mid) 88 | if fmid == flow { 89 | low = mid 90 | flow = fmid 91 | } else { 92 | high = mid 93 | fhigh = fmid 94 | } 95 | } 96 | } 97 | 98 | // series returns the sum of the series f(0), f(1), ... 99 | // 100 | // This implementation is fast, but subject to round-off error. 101 | func series(f func(float64) float64) float64 { 102 | y, yp := 0.0, 1.0 103 | for n := 0.0; y != yp; n++ { 104 | yp = y 105 | y += f(n) 106 | } 107 | return y 108 | } 109 | -------------------------------------------------------------------------------- /internal/stat/beta.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import "math" 4 | 5 | func lgamma(x float64) float64 { 6 | y, _ := math.Lgamma(x) 7 | return y 8 | } 9 | 10 | // mathBeta returns the value of the complete beta function B(a, b). 11 | func mathBeta(a, b float64) float64 { 12 | // B(x,y) = Γ(x)Γ(y) / Γ(x+y) 13 | return math.Exp(lgamma(a) + lgamma(b) - lgamma(a+b)) 14 | } 15 | 16 | // mathBetaInc returns the value of the regularized incomplete beta 17 | // function Iₓ(a, b). 18 | // 19 | // This is not to be confused with the "incomplete beta function", 20 | // which can be computed as BetaInc(x, a, b)*Beta(a, b). 21 | // 22 | // If x < 0 or x > 1, returns NaN. 23 | func mathBetaInc(x, a, b float64) float64 { 24 | // Based on Numerical Recipes in C, section 6.4. This uses the 25 | // continued fraction definition of I: 26 | // 27 | // (xᵃ*(1-x)ᵇ)/(a*B(a,b)) * (1/(1+(d₁/(1+(d₂/(1+...)))))) 28 | // 29 | // where B(a,b) is the beta function and 30 | // 31 | // d_{2m+1} = -(a+m)(a+b+m)x/((a+2m)(a+2m+1)) 32 | // d_{2m} = m(b-m)x/((a+2m-1)(a+2m)) 33 | if x < 0 || x > 1 { 34 | return math.NaN() 35 | } 36 | bt := 0.0 37 | if 0 < x && x < 1 { 38 | // Compute the coefficient before the continued 39 | // fraction. 40 | bt = math.Exp(lgamma(a+b) - lgamma(a) - lgamma(b) + 41 | a*math.Log(x) + b*math.Log(1-x)) 42 | } 43 | if x < (a+1)/(a+b+2) { 44 | // Compute continued fraction directly. 45 | return bt * betacf(x, a, b) / a 46 | } 47 | // Compute continued fraction after symmetry transform. 48 | return 1 - bt*betacf(1-x, b, a)/b 49 | } 50 | 51 | // betacf is the continued fraction component of the regularized 52 | // incomplete beta function Iₓ(a, b). 53 | func betacf(x, a, b float64) float64 { 54 | const maxIterations = 200 55 | const epsilon = 3e-14 56 | 57 | raiseZero := func(z float64) float64 { 58 | if math.Abs(z) < math.SmallestNonzeroFloat64 { 59 | return math.SmallestNonzeroFloat64 60 | } 61 | return z 62 | } 63 | 64 | c := 1.0 65 | d := 1 / raiseZero(1-(a+b)*x/(a+1)) 66 | h := d 67 | for m := 1; m <= maxIterations; m++ { 68 | mf := float64(m) 69 | 70 | // Even step of the recurrence. 71 | numer := mf * (b - mf) * x / ((a + 2*mf - 1) * (a + 2*mf)) 72 | d = 1 / raiseZero(1+numer*d) 73 | c = raiseZero(1 + numer/c) 74 | h *= d * c 75 | 76 | // Odd step of the recurrence. 77 | numer = -(a + mf) * (a + b + mf) * x / ((a + 2*mf) * (a + 2*mf + 1)) 78 | d = 1 / raiseZero(1+numer*d) 79 | c = raiseZero(1 + numer/c) 80 | hfac := d * c 81 | h *= hfac 82 | 83 | if math.Abs(hfac-1) < epsilon { 84 | return h 85 | } 86 | } 87 | panic("betainc: a or b too big; failed to converge") 88 | } 89 | -------------------------------------------------------------------------------- /internal/stat/data.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "strconv" 8 | "strings" 9 | 10 | "golang.design/x/bench/internal/benchfmt" 11 | "golang.design/x/bench/internal/term" 12 | ) 13 | 14 | // A Collection is a collection of benchmark results. 15 | type Collection struct { 16 | // Configs, Groups, and Units give the set of configs, 17 | // groups, and units from the keys in Stats in an order 18 | // meant to match the order the benchmarks were read in. 19 | Configs, Groups, Units []string 20 | 21 | // Benchmarks gives the set of benchmarks from the keys in 22 | // Stats by group in an order meant to match the order 23 | // benchmarks were read in. 24 | Benchmarks map[string][]string 25 | 26 | // Metrics holds the accumulated metrics for each key. 27 | Metrics map[Key]*Metrics 28 | 29 | // DeltaTest is the test to use to decide if a change is significant. 30 | // If nil, it defaults to UTest. 31 | DeltaTest DeltaTest 32 | 33 | // Alpha is the p-value cutoff to report a change as significant. 34 | // If zero, it defaults to 0.05. 35 | Alpha float64 36 | 37 | // AddGeoMean specifies whether to add a line to the table 38 | // showing the geometric mean of all the benchmark results. 39 | AddGeoMean bool 40 | 41 | // SplitBy specifies the labels to split results by. 42 | // By default, results will only be split by full name. 43 | SplitBy []string 44 | 45 | // Order specifies the row display order for this table. 46 | // If Order is nil, the table rows are printed in order of 47 | // first appearance in the input. 48 | Order Order 49 | } 50 | 51 | // A Key identifies one metric (e.g., "ns/op", "B/op") from one 52 | // benchmark (function name sans "Benchmark" prefix) and optional 53 | // group in one configuration (input file name). 54 | type Key struct { 55 | Config, Group, Benchmark, Unit string 56 | } 57 | 58 | // A Metrics holds the measurements of a single metric 59 | // (for example, ns/op or MB/s) 60 | // for all runs of a particular benchmark. 61 | type Metrics struct { 62 | Unit string // unit being measured 63 | Values []float64 // measured values 64 | RValues []float64 // Values with outliers removed 65 | Min float64 // min of RValues 66 | Mean float64 // mean of RValues 67 | Max float64 // max of RValues 68 | } 69 | 70 | // FormatMean formats m.Mean using scaler. 71 | func (m *Metrics) FormatMean(scaler Scaler) string { 72 | var s string 73 | if scaler != nil { 74 | s = scaler(m.Mean) 75 | } else { 76 | s = fmt.Sprint(m.Mean) 77 | } 78 | return s 79 | } 80 | 81 | // FormatDiff computes and formats the percent variation of max and min compared to mean. 82 | // If b.Mean or b.Max is zero, FormatDiff returns an empty string. 83 | func (m *Metrics) FormatDiff() string { 84 | if m.Mean == 0 || m.Max == 0 { 85 | return "" 86 | } 87 | diff := 1 - m.Min/m.Mean 88 | if d := m.Max/m.Mean - 1; d > diff { 89 | diff = d 90 | } 91 | 92 | if diff > 0.05 { 93 | return term.Orange(fmt.Sprintf("±%.0f%%", diff*100.0)) 94 | } 95 | return term.Gray(fmt.Sprintf("±%.0f%%", diff*100.0)) 96 | } 97 | 98 | // Format returns a textual formatting of "Mean ±Diff" using scaler. 99 | func (m *Metrics) Format(scaler Scaler) string { 100 | if m.Unit == "" { 101 | return "" 102 | } 103 | mean := m.FormatMean(scaler) 104 | diff := m.FormatDiff() 105 | if diff == "" { 106 | return mean + " " 107 | } 108 | return fmt.Sprintf("%s %3s", mean, diff) 109 | } 110 | 111 | // computeStats updates the derived statistics in m from the raw 112 | // samples in m.Values. 113 | func (m *Metrics) computeStats() { 114 | // Discard outliers. 115 | values := Sample{Xs: m.Values} 116 | q1, q3 := values.Percentile(0.25), values.Percentile(0.75) 117 | lo, hi := q1-1.5*(q3-q1), q3+1.5*(q3-q1) 118 | for _, value := range m.Values { 119 | if lo <= value && value <= hi { 120 | m.RValues = append(m.RValues, value) 121 | } 122 | } 123 | 124 | // Compute statistics of remaining data. 125 | m.Min, m.Max = Bounds(m.RValues) 126 | m.Mean = Mean(m.RValues) 127 | } 128 | 129 | // addMetrics returns the metrics with the given key from c, 130 | // creating a new one if needed. 131 | func (c *Collection) addMetrics(key Key) *Metrics { 132 | if c.Metrics == nil { 133 | c.Metrics = make(map[Key]*Metrics) 134 | } 135 | if stat, ok := c.Metrics[key]; ok { 136 | return stat 137 | } 138 | 139 | addString := func(strings *[]string, add string) { 140 | for _, s := range *strings { 141 | if s == add { 142 | return 143 | } 144 | } 145 | *strings = append(*strings, add) 146 | } 147 | addString(&c.Configs, key.Config) 148 | addString(&c.Groups, key.Group) 149 | if c.Benchmarks == nil { 150 | c.Benchmarks = make(map[string][]string) 151 | } 152 | benchmarks := c.Benchmarks[key.Group] 153 | addString(&benchmarks, key.Benchmark) 154 | c.Benchmarks[key.Group] = benchmarks 155 | addString(&c.Units, key.Unit) 156 | m := &Metrics{Unit: key.Unit} 157 | c.Metrics[key] = m 158 | return m 159 | } 160 | 161 | // AddFile adds the benchmark results in the formatted data 162 | // (read from the reader r) to the named configuration. 163 | func (c *Collection) AddFile(config string, f io.Reader) error { 164 | c.Configs = append(c.Configs, config) 165 | key := Key{Config: config} 166 | br := benchfmt.NewReader(f) 167 | for br.Next() { 168 | c.addResult(key, br.Result()) 169 | } 170 | return br.Err() 171 | } 172 | 173 | // AddData adds the benchmark results in the formatted data 174 | // (read from the reader r) to the named configuration. 175 | func (c *Collection) AddData(config string, data []byte) error { 176 | return c.AddFile(config, bytes.NewReader(data)) 177 | } 178 | 179 | // AddResults adds the benchmark results to the named configuration. 180 | func (c *Collection) AddResults(config string, results []*benchfmt.Result) { 181 | c.Configs = append(c.Configs, config) 182 | key := Key{Config: config} 183 | for _, r := range results { 184 | c.addResult(key, r) 185 | } 186 | } 187 | 188 | func (c *Collection) addResult(key Key, r *benchfmt.Result) { 189 | f := strings.Fields(r.Content) 190 | if len(f) < 4 { 191 | return 192 | } 193 | name := f[0] 194 | if !strings.HasPrefix(name, "Benchmark") { 195 | return 196 | } 197 | name = strings.TrimPrefix(name, "Benchmark") 198 | n, _ := strconv.Atoi(f[1]) 199 | if n == 0 { 200 | return 201 | } 202 | key.Group = c.makeGroup(r) 203 | key.Benchmark = name 204 | for i := 2; i+2 <= len(f); i += 2 { 205 | val, err := strconv.ParseFloat(f[i], 64) 206 | if err != nil { 207 | continue 208 | } 209 | key.Unit = f[i+1] 210 | m := c.addMetrics(key) 211 | m.Values = append(m.Values, val) 212 | } 213 | } 214 | 215 | func (c *Collection) makeGroup(r *benchfmt.Result) string { 216 | var out string 217 | for _, s := range c.SplitBy { 218 | v := r.NameLabels[s] 219 | if v == "" { 220 | v = r.Labels[s] 221 | } 222 | if v != "" { 223 | if out != "" { 224 | out = out + " " 225 | } 226 | out += fmt.Sprintf("%s:%s", s, v) 227 | } 228 | } 229 | return out 230 | } 231 | -------------------------------------------------------------------------------- /internal/stat/delta.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import "errors" 4 | 5 | // A DeltaTest compares the old and new metrics and returns the 6 | // expected probability that they are drawn from the same distribution. 7 | // 8 | // If a probability cannot be computed, the DeltaTest returns an 9 | // error explaining why. Common errors include ErrSamplesEqual 10 | // (all samples are equal), ErrSampleSize (there aren't enough samples), 11 | // and ErrZeroVariance (the sample has zero variance). 12 | // 13 | // As a special case, the missing test NoDeltaTest returns -1, nil. 14 | type DeltaTest func(old, new *Metrics) (float64, error) 15 | 16 | // Errors returned by DeltaTest. 17 | var ( 18 | ErrSamplesEqual = errors.New("all equal") 19 | ErrSampleSize = errors.New("too few samples") 20 | ErrZeroVariance = errors.New("zero variance") 21 | ErrMismatchedSamples = errors.New("samples have different lengths") 22 | ) 23 | 24 | // NoDeltaTest applies no delta test; it returns -1, nil. 25 | func NoDeltaTest(old, new *Metrics) (pval float64, err error) { 26 | return -1, nil 27 | } 28 | 29 | // TTest is a DeltaTest using the two-sample Welch t-test. 30 | func TTest(old, new *Metrics) (pval float64, err error) { 31 | t, err := TwoSampleWelchTTest( 32 | Sample{Xs: old.RValues}, 33 | Sample{Xs: new.RValues}, 34 | LocationDiffers, 35 | ) 36 | if err != nil { 37 | return -1, err 38 | } 39 | return t.P, nil 40 | } 41 | 42 | // UTest is a DeltaTest using the Mann-Whitney U test. 43 | func UTest(old, new *Metrics) (pval float64, err error) { 44 | u, err := MannWhitneyUTest(old.RValues, new.RValues, LocationDiffers) 45 | if err != nil { 46 | return -1, err 47 | } 48 | return u.P, nil 49 | } 50 | -------------------------------------------------------------------------------- /internal/stat/mathx.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import "math" 4 | 5 | var inf = math.Inf(1) 6 | var nan = math.NaN() 7 | 8 | // mathSign returns the sign of x: -1 if x < 0, 0 if x == 0, 1 if x > 0. 9 | // If x is NaN, it returns NaN. 10 | func mathSign(x float64) float64 { 11 | if x == 0 { 12 | return 0 13 | } else if x < 0 { 14 | return -1 15 | } else if x > 0 { 16 | return 1 17 | } 18 | return nan 19 | } 20 | 21 | const smallFactLimit = 20 // 20! => 62 bits 22 | var smallFact [smallFactLimit + 1]int64 23 | 24 | func init() { 25 | smallFact[0] = 1 26 | fact := int64(1) 27 | for n := int64(1); n <= smallFactLimit; n++ { 28 | fact *= n 29 | smallFact[n] = fact 30 | } 31 | } 32 | 33 | // mathChoose returns the binomial coefficient of n and k. 34 | func mathChoose(n, k int) float64 { 35 | if k == 0 || k == n { 36 | return 1 37 | } 38 | if k < 0 || n < k { 39 | return 0 40 | } 41 | if n <= smallFactLimit { // Implies k <= smallFactLimit 42 | // It's faster to do several integer multiplications 43 | // than it is to do an extra integer division. 44 | // Remarkably, this is also faster than pre-computing 45 | // Pascal's triangle (presumably because this is very 46 | // cache efficient). 47 | numer := int64(1) 48 | for n1 := int64(n - (k - 1)); n1 <= int64(n); n1++ { 49 | numer *= n1 50 | } 51 | denom := smallFact[k] 52 | return float64(numer / denom) 53 | } 54 | 55 | return math.Exp(lchoose(n, k)) 56 | } 57 | 58 | // mathLchoose returns math.Log(mathChoose(n, k)). 59 | func mathLchoose(n, k int) float64 { 60 | if k == 0 || k == n { 61 | return 0 62 | } 63 | if k < 0 || n < k { 64 | return math.NaN() 65 | } 66 | return lchoose(n, k) 67 | } 68 | 69 | func lchoose(n, k int) float64 { 70 | a, _ := math.Lgamma(float64(n + 1)) 71 | b, _ := math.Lgamma(float64(k + 1)) 72 | c, _ := math.Lgamma(float64(n - k + 1)) 73 | return a - b - c 74 | } 75 | -------------------------------------------------------------------------------- /internal/stat/normaldist.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "math" 5 | "math/rand" 6 | ) 7 | 8 | // NormalDist is a normal (Gaussian) distribution with mean Mu and 9 | // standard deviation Sigma. 10 | type NormalDist struct { 11 | Mu, Sigma float64 12 | } 13 | 14 | // StdNormal is the standard normal distribution (Mu = 0, Sigma = 1) 15 | var StdNormal = NormalDist{0, 1} 16 | 17 | // 1/sqrt(2 * pi) 18 | const invSqrt2Pi = 0.39894228040143267793994605993438186847585863116493465766592583 19 | 20 | func (n NormalDist) PDF(x float64) float64 { 21 | z := x - n.Mu 22 | return math.Exp(-z*z/(2*n.Sigma*n.Sigma)) * invSqrt2Pi / n.Sigma 23 | } 24 | 25 | func (n NormalDist) pdfEach(xs []float64) []float64 { 26 | res := make([]float64, len(xs)) 27 | if n.Mu == 0 && n.Sigma == 1 { 28 | // Standard normal fast path 29 | for i, x := range xs { 30 | res[i] = math.Exp(-x*x/2) * invSqrt2Pi 31 | } 32 | } else { 33 | a := -1 / (2 * n.Sigma * n.Sigma) 34 | b := invSqrt2Pi / n.Sigma 35 | for i, x := range xs { 36 | z := x - n.Mu 37 | res[i] = math.Exp(z*z*a) * b 38 | } 39 | } 40 | return res 41 | } 42 | 43 | func (n NormalDist) CDF(x float64) float64 { 44 | return math.Erfc(-(x-n.Mu)/(n.Sigma*math.Sqrt2)) / 2 45 | } 46 | 47 | func (n NormalDist) cdfEach(xs []float64) []float64 { 48 | res := make([]float64, len(xs)) 49 | a := 1 / (n.Sigma * math.Sqrt2) 50 | for i, x := range xs { 51 | res[i] = math.Erfc(-(x-n.Mu)*a) / 2 52 | } 53 | return res 54 | } 55 | 56 | func (n NormalDist) InvCDF(p float64) (x float64) { 57 | // This is based on Peter John Acklam's inverse normal CDF 58 | // algorithm: http://home.online.no/~pjacklam/notes/invnorm/ 59 | const ( 60 | a1 = -3.969683028665376e+01 61 | a2 = 2.209460984245205e+02 62 | a3 = -2.759285104469687e+02 63 | a4 = 1.383577518672690e+02 64 | a5 = -3.066479806614716e+01 65 | a6 = 2.506628277459239e+00 66 | 67 | b1 = -5.447609879822406e+01 68 | b2 = 1.615858368580409e+02 69 | b3 = -1.556989798598866e+02 70 | b4 = 6.680131188771972e+01 71 | b5 = -1.328068155288572e+01 72 | 73 | c1 = -7.784894002430293e-03 74 | c2 = -3.223964580411365e-01 75 | c3 = -2.400758277161838e+00 76 | c4 = -2.549732539343734e+00 77 | c5 = 4.374664141464968e+00 78 | c6 = 2.938163982698783e+00 79 | 80 | d1 = 7.784695709041462e-03 81 | d2 = 3.224671290700398e-01 82 | d3 = 2.445134137142996e+00 83 | d4 = 3.754408661907416e+00 84 | 85 | plow = 0.02425 86 | phigh = 1 - plow 87 | ) 88 | 89 | if p < 0 || p > 1 { 90 | return nan 91 | } else if p == 0 { 92 | return -inf 93 | } else if p == 1 { 94 | return inf 95 | } 96 | 97 | if p < plow { 98 | // Rational approximation for lower region. 99 | q := math.Sqrt(-2 * math.Log(p)) 100 | x = (((((c1*q+c2)*q+c3)*q+c4)*q+c5)*q + c6) / 101 | ((((d1*q+d2)*q+d3)*q+d4)*q + 1) 102 | } else if phigh < p { 103 | // Rational approximation for upper region. 104 | q := math.Sqrt(-2 * math.Log(1-p)) 105 | x = -(((((c1*q+c2)*q+c3)*q+c4)*q+c5)*q + c6) / 106 | ((((d1*q+d2)*q+d3)*q+d4)*q + 1) 107 | } else { 108 | // Rational approximation for central region. 109 | q := p - 0.5 110 | r := q * q 111 | x = (((((a1*r+a2)*r+a3)*r+a4)*r+a5)*r + a6) * q / 112 | (((((b1*r+b2)*r+b3)*r+b4)*r+b5)*r + 1) 113 | } 114 | 115 | // Refine approximation. 116 | e := 0.5*math.Erfc(-x/math.Sqrt2) - p 117 | u := e * math.Sqrt(2*math.Pi) * math.Exp(x*x/2) 118 | x = x - u/(1+x*u/2) 119 | 120 | // Adjust from standard normal. 121 | return x*n.Sigma + n.Mu 122 | } 123 | 124 | func (n NormalDist) Rand(r *rand.Rand) float64 { 125 | var x float64 126 | if r == nil { 127 | x = rand.NormFloat64() 128 | } else { 129 | x = r.NormFloat64() 130 | } 131 | return x*n.Sigma + n.Mu 132 | } 133 | 134 | func (n NormalDist) Bounds() (float64, float64) { 135 | const stddevs = 3 136 | return n.Mu - stddevs*n.Sigma, n.Mu + stddevs*n.Sigma 137 | } 138 | -------------------------------------------------------------------------------- /internal/stat/sample.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "math" 5 | "sort" 6 | ) 7 | 8 | // Sample is a collection of possibly weighted data points. 9 | type Sample struct { 10 | // Xs is the slice of sample values. 11 | Xs []float64 12 | 13 | // Weights[i] is the weight of sample Xs[i]. If Weights is 14 | // nil, all Xs have weight 1. Weights must have the same 15 | // length of Xs and all values must be non-negative. 16 | Weights []float64 17 | 18 | // Sorted indicates that Xs is sorted in ascending order. 19 | Sorted bool 20 | } 21 | 22 | // Bounds returns the minimum and maximum values of xs. 23 | func Bounds(xs []float64) (min float64, max float64) { 24 | if len(xs) == 0 { 25 | return math.NaN(), math.NaN() 26 | } 27 | min, max = xs[0], xs[0] 28 | for _, x := range xs { 29 | if x < min { 30 | min = x 31 | } 32 | if x > max { 33 | max = x 34 | } 35 | } 36 | return 37 | } 38 | 39 | // Bounds returns the minimum and maximum values of the Sample. 40 | // 41 | // If the Sample is weighted, this ignores samples with zero weight. 42 | // 43 | // This is constant time if s.Sorted and there are no zero-weighted 44 | // values. 45 | func (s Sample) Bounds() (min float64, max float64) { 46 | if len(s.Xs) == 0 || (!s.Sorted && s.Weights == nil) { 47 | return Bounds(s.Xs) 48 | } 49 | 50 | if s.Sorted { 51 | if s.Weights == nil { 52 | return s.Xs[0], s.Xs[len(s.Xs)-1] 53 | } 54 | min, max = math.NaN(), math.NaN() 55 | for i, w := range s.Weights { 56 | if w != 0 { 57 | min = s.Xs[i] 58 | break 59 | } 60 | } 61 | if math.IsNaN(min) { 62 | return 63 | } 64 | for i := range s.Weights { 65 | if s.Weights[len(s.Weights)-i-1] != 0 { 66 | max = s.Xs[len(s.Weights)-i-1] 67 | break 68 | } 69 | } 70 | } else { 71 | min, max = math.Inf(1), math.Inf(-1) 72 | for i, x := range s.Xs { 73 | w := s.Weights[i] 74 | if x < min && w != 0 { 75 | min = x 76 | } 77 | if x > max && w != 0 { 78 | max = x 79 | } 80 | } 81 | if math.IsInf(min, 0) { 82 | min, max = math.NaN(), math.NaN() 83 | } 84 | } 85 | return 86 | } 87 | 88 | // vecSum returns the sum of xs. 89 | func vecSum(xs []float64) float64 { 90 | sum := 0.0 91 | for _, x := range xs { 92 | sum += x 93 | } 94 | return sum 95 | } 96 | 97 | // Sum returns the (possibly weighted) sum of the Sample. 98 | func (s Sample) Sum() float64 { 99 | if s.Weights == nil { 100 | return vecSum(s.Xs) 101 | } 102 | sum := 0.0 103 | for i, x := range s.Xs { 104 | sum += x * s.Weights[i] 105 | } 106 | return sum 107 | } 108 | 109 | // Weight returns the total weight of the Sasmple. 110 | func (s Sample) Weight() float64 { 111 | if s.Weights == nil { 112 | return float64(len(s.Xs)) 113 | } 114 | return vecSum(s.Weights) 115 | } 116 | 117 | // Mean returns the arithmetic mean of xs. 118 | func Mean(xs []float64) float64 { 119 | if len(xs) == 0 { 120 | return math.NaN() 121 | } 122 | m := 0.0 123 | for i, x := range xs { 124 | m += (x - m) / float64(i+1) 125 | } 126 | return m 127 | } 128 | 129 | // Mean returns the arithmetic mean of the Sample. 130 | func (s Sample) Mean() float64 { 131 | if len(s.Xs) == 0 || s.Weights == nil { 132 | return Mean(s.Xs) 133 | } 134 | 135 | m, wsum := 0.0, 0.0 136 | for i, x := range s.Xs { 137 | // Use weighted incremental mean: 138 | // m_i = (1 - w_i/wsum_i) * m_(i-1) + (w_i/wsum_i) * x_i 139 | // = m_(i-1) + (x_i - m_(i-1)) * (w_i/wsum_i) 140 | w := s.Weights[i] 141 | wsum += w 142 | m += (x - m) * w / wsum 143 | } 144 | return m 145 | } 146 | 147 | // GeoMean returns the geometric mean of xs. xs must be positive. 148 | func GeoMean(xs []float64) float64 { 149 | if len(xs) == 0 { 150 | return math.NaN() 151 | } 152 | m := 0.0 153 | for i, x := range xs { 154 | if x <= 0 { 155 | return math.NaN() 156 | } 157 | lx := math.Log(x) 158 | m += (lx - m) / float64(i+1) 159 | } 160 | return math.Exp(m) 161 | } 162 | 163 | // GeoMean returns the geometric mean of the Sample. All samples 164 | // values must be positive. 165 | func (s Sample) GeoMean() float64 { 166 | if len(s.Xs) == 0 || s.Weights == nil { 167 | return GeoMean(s.Xs) 168 | } 169 | 170 | m, wsum := 0.0, 0.0 171 | for i, x := range s.Xs { 172 | w := s.Weights[i] 173 | wsum += w 174 | lx := math.Log(x) 175 | m += (lx - m) * w / wsum 176 | } 177 | return math.Exp(m) 178 | } 179 | 180 | // Variance returns the sample variance of xs. 181 | func Variance(xs []float64) float64 { 182 | if len(xs) == 0 { 183 | return math.NaN() 184 | } else if len(xs) <= 1 { 185 | return 0 186 | } 187 | 188 | // Based on Wikipedia's presentation of Welford 1962 189 | // (http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm). 190 | // This is more numerically stable than the standard two-pass 191 | // formula and not prone to massive cancellation. 192 | mean, M2 := 0.0, 0.0 193 | for n, x := range xs { 194 | delta := x - mean 195 | mean += delta / float64(n+1) 196 | M2 += delta * (x - mean) 197 | } 198 | return M2 / float64(len(xs)-1) 199 | } 200 | 201 | // Variance returns the sample variance of xs. 202 | func (s Sample) Variance() float64 { 203 | if len(s.Xs) == 0 || s.Weights == nil { 204 | return Variance(s.Xs) 205 | } 206 | // TODO(austin) 207 | panic("Weighted Variance not implemented") 208 | } 209 | 210 | // StdDev returns the sample standard deviation of xs. 211 | func StdDev(xs []float64) float64 { 212 | return math.Sqrt(Variance(xs)) 213 | } 214 | 215 | // StdDev returns the sample standard deviation of the Sample. 216 | func (s Sample) StdDev() float64 { 217 | if len(s.Xs) == 0 || s.Weights == nil { 218 | return StdDev(s.Xs) 219 | } 220 | // TODO(austin) 221 | panic("Weighted StdDev not implemented") 222 | } 223 | 224 | // Percentile returns the pctileth value from the Sample. This uses 225 | // interpolation method R8 from Hyndman and Fan (1996). 226 | // 227 | // pctile will be capped to the range [0, 1]. If len(xs) == 0 or all 228 | // weights are 0, returns NaN. 229 | // 230 | // Percentile(0.5) is the median. Percentile(0.25) and 231 | // Percentile(0.75) are the first and third quartiles, respectively. 232 | // 233 | // This is constant time if s.Sorted and s.Weights == nil. 234 | func (s Sample) Percentile(pctile float64) float64 { 235 | if len(s.Xs) == 0 { 236 | return math.NaN() 237 | } else if pctile <= 0 { 238 | min, _ := s.Bounds() 239 | return min 240 | } else if pctile >= 1 { 241 | _, max := s.Bounds() 242 | return max 243 | } 244 | 245 | if !s.Sorted { 246 | // TODO(austin) Use select algorithm instead 247 | s = *s.Copy().Sort() 248 | } 249 | 250 | if s.Weights == nil { 251 | N := float64(len(s.Xs)) 252 | //n := pctile * (N + 1) // R6 253 | n := 1/3.0 + pctile*(N+1/3.0) // R8 254 | kf, frac := math.Modf(n) 255 | k := int(kf) 256 | if k <= 0 { 257 | return s.Xs[0] 258 | } else if k >= len(s.Xs) { 259 | return s.Xs[len(s.Xs)-1] 260 | } 261 | return s.Xs[k-1] + frac*(s.Xs[k]-s.Xs[k-1]) 262 | } 263 | // TODO(austin): Implement interpolation 264 | 265 | target := s.Weight() * pctile 266 | 267 | // TODO(austin) If we had cumulative weights, we could 268 | // do this in log time. 269 | for i, weight := range s.Weights { 270 | target -= weight 271 | if target < 0 { 272 | return s.Xs[i] 273 | } 274 | } 275 | return s.Xs[len(s.Xs)-1] 276 | } 277 | 278 | // IQR returns the interquartile range of the Sample. 279 | // 280 | // This is constant time if s.Sorted and s.Weights == nil. 281 | func (s Sample) IQR() float64 { 282 | if !s.Sorted { 283 | s = *s.Copy().Sort() 284 | } 285 | return s.Percentile(0.75) - s.Percentile(0.25) 286 | } 287 | 288 | type sampleSorter struct { 289 | xs []float64 290 | weights []float64 291 | } 292 | 293 | func (p *sampleSorter) Len() int { 294 | return len(p.xs) 295 | } 296 | 297 | func (p *sampleSorter) Less(i, j int) bool { 298 | return p.xs[i] < p.xs[j] 299 | } 300 | 301 | func (p *sampleSorter) Swap(i, j int) { 302 | p.xs[i], p.xs[j] = p.xs[j], p.xs[i] 303 | p.weights[i], p.weights[j] = p.weights[j], p.weights[i] 304 | } 305 | 306 | // Sort sorts the samples in place in s and returns s. 307 | // 308 | // A sorted sample improves the performance of some algorithms. 309 | func (s *Sample) Sort() *Sample { 310 | if s.Sorted || sort.Float64sAreSorted(s.Xs) { 311 | // All set 312 | } else if s.Weights == nil { 313 | sort.Float64s(s.Xs) 314 | } else { 315 | sort.Sort(&sampleSorter{s.Xs, s.Weights}) 316 | } 317 | s.Sorted = true 318 | return s 319 | } 320 | 321 | // Copy returns a copy of the Sample. 322 | // 323 | // The returned Sample shares no data with the original, so they can 324 | // be modified (for example, sorted) independently. 325 | func (s Sample) Copy() *Sample { 326 | xs := make([]float64, len(s.Xs)) 327 | copy(xs, s.Xs) 328 | 329 | weights := []float64(nil) 330 | if s.Weights != nil { 331 | weights = make([]float64, len(s.Weights)) 332 | copy(weights, s.Weights) 333 | } 334 | 335 | return &Sample{xs, weights, s.Sorted} 336 | } 337 | -------------------------------------------------------------------------------- /internal/stat/scaler.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // A Scaler is a function that scales and formats a measurement. 9 | // All measurements within a given table row are formatted 10 | // using the same scaler, so that the units are consistent 11 | // across the row. 12 | type Scaler func(float64) string 13 | 14 | // NewScaler returns a Scaler appropriate for formatting 15 | // the measurement val, which has the given unit. 16 | func NewScaler(val float64, unit string) Scaler { 17 | if hasBaseUnit(unit, "ns/op") || hasBaseUnit(unit, "ns/GC") { 18 | return timeScaler(val) 19 | } 20 | 21 | var format string 22 | var scale float64 23 | var suffix string 24 | 25 | prescale := 1.0 26 | if hasBaseUnit(unit, "MB/s") { 27 | prescale = 1e6 28 | } 29 | 30 | switch x := val * prescale; { 31 | case x >= 99500000000000: 32 | format, scale, suffix = "%.0f", 1e12, "T" 33 | case x >= 9950000000000: 34 | format, scale, suffix = "%.1f", 1e12, "T" 35 | case x >= 995000000000: 36 | format, scale, suffix = "%.2f", 1e12, "T" 37 | case x >= 99500000000: 38 | format, scale, suffix = "%.0f", 1e9, "G" 39 | case x >= 9950000000: 40 | format, scale, suffix = "%.1f", 1e9, "G" 41 | case x >= 995000000: 42 | format, scale, suffix = "%.2f", 1e9, "G" 43 | case x >= 99500000: 44 | format, scale, suffix = "%.0f", 1e6, "M" 45 | case x >= 9950000: 46 | format, scale, suffix = "%.1f", 1e6, "M" 47 | case x >= 995000: 48 | format, scale, suffix = "%.2f", 1e6, "M" 49 | case x >= 99500: 50 | format, scale, suffix = "%.0f", 1e3, "k" 51 | case x >= 9950: 52 | format, scale, suffix = "%.1f", 1e3, "k" 53 | case x >= 995: 54 | format, scale, suffix = "%.2f", 1e3, "k" 55 | case x >= 99.5: 56 | format, scale, suffix = "%.0f", 1, "" 57 | case x >= 9.95: 58 | format, scale, suffix = "%.1f", 1, "" 59 | default: 60 | format, scale, suffix = "%.2f", 1, "" 61 | } 62 | 63 | if hasBaseUnit(unit, "B/op") || hasBaseUnit(unit, "bytes/op") || hasBaseUnit(unit, "bytes") { 64 | suffix += "B" 65 | } 66 | if hasBaseUnit(unit, "MB/s") { 67 | suffix += "B/s" 68 | } 69 | scale /= prescale 70 | 71 | return func(val float64) string { 72 | return fmt.Sprintf(format+suffix, val/scale) 73 | } 74 | } 75 | 76 | func timeScaler(ns float64) Scaler { 77 | var format string 78 | var scale float64 79 | switch x := ns / 1e9; { 80 | case x >= 99.5: 81 | format, scale = "%.0fs", 1 82 | case x >= 9.95: 83 | format, scale = "%.1fs", 1 84 | case x >= 0.995: 85 | format, scale = "%.2fs", 1 86 | case x >= 0.0995: 87 | format, scale = "%.0fms", 1000 88 | case x >= 0.00995: 89 | format, scale = "%.1fms", 1000 90 | case x >= 0.000995: 91 | format, scale = "%.2fms", 1000 92 | case x >= 0.0000995: 93 | format, scale = "%.0fµs", 1000*1000 94 | case x >= 0.00000995: 95 | format, scale = "%.1fµs", 1000*1000 96 | case x >= 0.000000995: 97 | format, scale = "%.2fµs", 1000*1000 98 | case x >= 0.0000000995: 99 | format, scale = "%.0fns", 1000*1000*1000 100 | case x >= 0.00000000995: 101 | format, scale = "%.1fns", 1000*1000*1000 102 | default: 103 | format, scale = "%.2fns", 1000*1000*1000 104 | } 105 | return func(ns float64) string { 106 | return fmt.Sprintf(format, ns/1e9*scale) 107 | } 108 | } 109 | 110 | // hasBaseUnit reports whether s has unit unit. 111 | // For now, it reports whether s == unit or s ends in -unit. 112 | func hasBaseUnit(s, unit string) bool { 113 | return s == unit || strings.HasSuffix(s, "-"+unit) 114 | } 115 | -------------------------------------------------------------------------------- /internal/stat/sort.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "math" 5 | "sort" 6 | ) 7 | 8 | // An Order defines a sort order for a table. 9 | // It reports whether t.Rows[i] should appear before t.Rows[j]. 10 | type Order func(t *Table, i, j int) bool 11 | 12 | // ByName sorts tables by the Benchmark name column 13 | func ByName(t *Table, i, j int) bool { 14 | return t.Rows[i].Benchmark < t.Rows[j].Benchmark 15 | } 16 | 17 | // ByDelta sorts tables by the Delta column, 18 | // reversing the order when larger is better (for "speed" results). 19 | func ByDelta(t *Table, i, j int) bool { 20 | return math.Abs(t.Rows[i].PctDelta)*float64(t.Rows[i].Change) < 21 | math.Abs(t.Rows[j].PctDelta)*float64(t.Rows[j].Change) 22 | } 23 | 24 | // Reverse returns the reverse of the given order. 25 | func Reverse(order Order) Order { 26 | return func(t *Table, i, j int) bool { return order(t, j, i) } 27 | } 28 | 29 | // Sort sorts a Table t (in place) by the given order. 30 | func Sort(t *Table, order Order) { 31 | sort.SliceStable(t.Rows, func(i, j int) bool { return order(t, i, j) }) 32 | } 33 | -------------------------------------------------------------------------------- /internal/stat/table.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // A Table is a table for display in the benchstat output. 9 | type Table struct { 10 | Metric string 11 | OldNewDelta bool // is this an old-new-delta table? 12 | Configs []string 13 | Groups []string 14 | Rows []*Row 15 | } 16 | 17 | // A Row is a table row for display in the benchstat output. 18 | type Row struct { 19 | Benchmark string // benchmark name 20 | Group string // group name 21 | Scaler Scaler // formatter for stats means 22 | Metrics []*Metrics // columns of statistics 23 | PctDelta float64 // unformatted percent change 24 | Delta string // formatted percent change 25 | Note string // additional information 26 | Change int // +1 better, -1 worse, 0 unchanged 27 | } 28 | 29 | // Tables returns tables comparing the benchmarks in the collection. 30 | func (c *Collection) Tables() []*Table { 31 | deltaTest := c.DeltaTest 32 | if deltaTest == nil { 33 | deltaTest = UTest 34 | } 35 | alpha := c.Alpha 36 | if alpha == 0 { 37 | alpha = 0.05 38 | } 39 | 40 | // Update statistics. 41 | for _, m := range c.Metrics { 42 | m.computeStats() 43 | } 44 | 45 | var tables []*Table 46 | key := Key{} 47 | for _, key.Unit = range c.Units { 48 | table := new(Table) 49 | table.Configs = c.Configs 50 | table.Groups = c.Groups 51 | table.Metric = metricOf(key.Unit) 52 | table.OldNewDelta = len(c.Configs) == 2 53 | for _, key.Group = range c.Groups { 54 | for _, key.Benchmark = range c.Benchmarks[key.Group] { 55 | row := &Row{Benchmark: key.Benchmark} 56 | if len(c.Groups) > 1 { 57 | // Show group headers if there is more than one group. 58 | row.Group = key.Group 59 | } 60 | 61 | for _, key.Config = range c.Configs { 62 | m := c.Metrics[key] 63 | if m == nil { 64 | row.Metrics = append(row.Metrics, new(Metrics)) 65 | continue 66 | } 67 | row.Metrics = append(row.Metrics, m) 68 | if row.Scaler == nil { 69 | row.Scaler = NewScaler(m.Mean, m.Unit) 70 | } 71 | } 72 | 73 | // If there are only two configs being compared, add stats. 74 | if table.OldNewDelta { 75 | k0 := key 76 | k0.Config = c.Configs[0] 77 | k1 := key 78 | k1.Config = c.Configs[1] 79 | old := c.Metrics[k0] 80 | new := c.Metrics[k1] 81 | // If one is missing, omit row entirely. 82 | // TODO: Control this better. 83 | if old == nil || new == nil { 84 | continue 85 | } 86 | pval, testerr := deltaTest(old, new) 87 | row.PctDelta = 0.00 88 | row.Delta = "~" 89 | if testerr == ErrZeroVariance { 90 | row.Note = "(zero variance)" 91 | } else if testerr == ErrSampleSize { 92 | row.Note = "(too few samples)" 93 | } else if testerr == ErrSamplesEqual { 94 | row.Note = "(all equal)" 95 | } else if testerr != nil { 96 | row.Note = fmt.Sprintf("(%s)", testerr) 97 | } else if pval < alpha { 98 | if new.Mean == old.Mean { 99 | row.Delta = "0.00%" 100 | } else { 101 | pct := ((new.Mean / old.Mean) - 1.0) * 100.0 102 | row.PctDelta = pct 103 | row.Delta = fmt.Sprintf("%+.2f%%", pct) 104 | if pct < 0 == (table.Metric != "speed") { // smaller is better, except speeds 105 | row.Change = +1 106 | } else { 107 | row.Change = -1 108 | } 109 | } 110 | } 111 | if row.Note == "" && pval != -1 { 112 | row.Note = fmt.Sprintf("(p=%0.3f n=%d+%d)", pval, len(old.RValues), len(new.RValues)) 113 | } 114 | } 115 | 116 | table.Rows = append(table.Rows, row) 117 | } 118 | } 119 | 120 | if len(table.Rows) > 0 { 121 | if c.Order != nil { 122 | Sort(table, c.Order) 123 | } 124 | if c.AddGeoMean { 125 | addGeomean(c, table, key.Unit, table.OldNewDelta) 126 | } 127 | tables = append(tables, table) 128 | } 129 | } 130 | 131 | return tables 132 | } 133 | 134 | var metricSuffix = map[string]string{ 135 | "ns/op": "time/op", 136 | "ns/GC": "time/GC", 137 | "B/op": "alloc/op", 138 | "MB/s": "speed", 139 | } 140 | 141 | // metricOf returns the name of the metric with the given unit. 142 | func metricOf(unit string) string { 143 | if s := metricSuffix[unit]; s != "" { 144 | return s 145 | } 146 | for s, suff := range metricSuffix { 147 | if dashs := "-" + s; strings.HasSuffix(unit, dashs) { 148 | prefix := strings.TrimSuffix(unit, dashs) 149 | return prefix + "-" + suff 150 | } 151 | } 152 | return unit 153 | } 154 | 155 | // addGeomean adds a "geomean" row to the table, 156 | // showing the geometric mean of all the benchmarks. 157 | func addGeomean(c *Collection, t *Table, unit string, delta bool) { 158 | row := &Row{Benchmark: "[Geo mean]"} 159 | key := Key{Unit: unit} 160 | geomeans := []float64{} 161 | maxCount := 0 162 | for _, key.Config = range c.Configs { 163 | var means []float64 164 | for _, key.Group = range c.Groups { 165 | for _, key.Benchmark = range c.Benchmarks[key.Group] { 166 | m := c.Metrics[key] 167 | // Omit 0 values from the geomean calculation, 168 | // as these either make the geomean undefined 169 | // or zero (depending on who you ask). This 170 | // typically comes up with things like 171 | // allocation counts, where it's fine to just 172 | // ignore the benchmark. 173 | if m != nil && m.Mean != 0 { 174 | means = append(means, m.Mean) 175 | } 176 | } 177 | } 178 | if len(means) > maxCount { 179 | maxCount = len(means) 180 | } 181 | if len(means) == 0 { 182 | row.Metrics = append(row.Metrics, new(Metrics)) 183 | delta = false 184 | } else { 185 | geomean := GeoMean(means) 186 | geomeans = append(geomeans, geomean) 187 | if row.Scaler == nil { 188 | row.Scaler = NewScaler(geomean, unit) 189 | } 190 | row.Metrics = append(row.Metrics, &Metrics{ 191 | Unit: unit, 192 | Mean: geomean, 193 | }) 194 | } 195 | } 196 | if maxCount <= 1 { 197 | // Only one benchmark contributed to this geomean. 198 | // Since the geomean is the same as the benchmark 199 | // result, don't bother outputting it. 200 | return 201 | } 202 | if delta { 203 | pct := ((geomeans[1] / geomeans[0]) - 1.0) * 100.0 204 | row.PctDelta = pct 205 | row.Delta = fmt.Sprintf("%+.2f%%", pct) 206 | } 207 | t.Rows = append(t.Rows, row) 208 | } 209 | -------------------------------------------------------------------------------- /internal/stat/tdist.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import "math" 4 | 5 | // A TDist is a Student's t-distribution with V degrees of freedom. 6 | type TDist struct { 7 | V float64 8 | } 9 | 10 | // PDF is the probability distribution function 11 | func (t TDist) PDF(x float64) float64 { 12 | return math.Exp(lgamma((t.V+1)/2)-lgamma(t.V/2)) / 13 | math.Sqrt(t.V*math.Pi) * math.Pow(1+(x*x)/t.V, -(t.V+1)/2) 14 | } 15 | 16 | // CDF is the cumulative distribution function 17 | func (t TDist) CDF(x float64) float64 { 18 | if x == 0 { 19 | return 0.5 20 | } else if x > 0 { 21 | return 1 - 0.5*mathBetaInc(t.V/(t.V+x*x), t.V/2, 0.5) 22 | } else if x < 0 { 23 | return 1 - t.CDF(-x) 24 | } else { 25 | return math.NaN() 26 | } 27 | } 28 | 29 | // Bounds ... 30 | func (t TDist) Bounds() (float64, float64) { 31 | return -4, 4 32 | } 33 | -------------------------------------------------------------------------------- /internal/stat/text.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "unicode/utf8" 7 | 8 | "golang.design/x/bench/internal/term" 9 | ) 10 | 11 | // FormatText appends a fixed-width text formatting of the tables to w. 12 | func FormatText(w io.Writer, tables []*Table) { 13 | var textTables [][]*textRow 14 | for _, t := range tables { 15 | textTables = append(textTables, toText(t, true)) 16 | } 17 | 18 | var max []int 19 | for _, table := range textTables { 20 | for _, row := range table { 21 | if len(row.cols) == 1 { 22 | // Header row 23 | continue 24 | } 25 | for len(max) < len(row.cols) { 26 | max = append(max, 0) 27 | } 28 | for i, s := range row.cols { 29 | n := utf8.RuneCountInString(s) 30 | if max[i] < n { 31 | max[i] = n 32 | } 33 | } 34 | } 35 | } 36 | 37 | for i, table := range textTables { 38 | if i > 0 { 39 | fmt.Fprintf(w, "\n") 40 | } 41 | 42 | // headings 43 | row := table[0] 44 | for i, s := range row.cols { 45 | switch i { 46 | case 0: 47 | fmt.Fprintf(w, "%-*s", max[i], s) 48 | default: 49 | fmt.Fprintf(w, " %-*s", max[i], s) 50 | case len(row.cols) - 1: 51 | fmt.Fprintf(w, " %s\n", s) 52 | } 53 | } 54 | 55 | // data 56 | for _, row := range table[1:] { 57 | for i, s := range row.cols { 58 | switch { 59 | case len(row.cols) == 1: 60 | // Header row 61 | fmt.Fprint(w, s) 62 | case i == 0: 63 | fmt.Fprintf(w, "%-*s", max[i], s) 64 | default: 65 | if i == len(row.cols)-1 && len(s) > 0 && s[0] == '(' { 66 | // Left-align p value. 67 | fmt.Fprintf(w, " %s", s) 68 | break 69 | } 70 | fmt.Fprintf(w, " %*s", max[i], s) 71 | } 72 | } 73 | fmt.Fprintf(w, "\n") 74 | } 75 | } 76 | } 77 | 78 | // A textRow is a row of printed text columns. 79 | type textRow struct { 80 | cols []string 81 | } 82 | 83 | func newTextRow(cols ...string) *textRow { 84 | return &textRow{cols: cols} 85 | } 86 | 87 | // newTextRowDelta returns a labeled row of text, with "±" inserted after 88 | // each member of "cols" unless norange is true. 89 | func newTextRowDelta(norange bool, label string, cols ...string) *textRow { 90 | newcols := []string{} 91 | newcols = append(newcols, label) 92 | for _, s := range cols { 93 | newcols = append(newcols, s) 94 | if !norange { 95 | newcols = append(newcols, "±") 96 | } 97 | } 98 | return &textRow{cols: newcols} 99 | } 100 | 101 | func (r *textRow) add(col string) { 102 | r.cols = append(r.cols, col) 103 | } 104 | 105 | func (r *textRow) trim() { 106 | for len(r.cols) > 0 && r.cols[len(r.cols)-1] == "" { 107 | r.cols = r.cols[:len(r.cols)-1] 108 | } 109 | } 110 | 111 | // toText converts the Table to a textual grid of cells, 112 | // which can then be printed in fixed-width output. 113 | func toText(t *Table, colorful bool) []*textRow { 114 | var textRows []*textRow 115 | switch len(t.Configs) { 116 | case 1: 117 | textRows = append(textRows, newTextRow("name", t.Metric)) 118 | case 2: 119 | textRows = append(textRows, newTextRow("name", "old "+t.Metric, "new "+t.Metric, "delta")) 120 | default: 121 | row := newTextRow("name \\ " + t.Metric) 122 | row.cols = append(row.cols, t.Configs...) // TODO Should this also trim common path prefix? (see toCSV) 123 | textRows = append(textRows, row) 124 | } 125 | 126 | var group string 127 | 128 | for _, row := range t.Rows { 129 | if row.Group != group { 130 | group = row.Group 131 | textRows = append(textRows, newTextRow(group)) 132 | } 133 | text := newTextRow(row.Benchmark) 134 | for _, m := range row.Metrics { 135 | text.cols = append(text.cols, m.Format(row.Scaler)) 136 | } 137 | if len(t.Configs) == 2 { 138 | delta := row.Delta 139 | if delta == "~" { 140 | delta = "~ " 141 | } 142 | if colorful { 143 | switch row.Change { 144 | case 1: // better 145 | delta = term.Green(delta) 146 | case -1: // worse 147 | delta = term.Red(delta) 148 | default: // no change 149 | delta = term.Gray(delta) 150 | } 151 | } 152 | text.cols = append(text.cols, delta) 153 | text.cols = append(text.cols, row.Note) 154 | } 155 | textRows = append(textRows, text) 156 | } 157 | for _, r := range textRows { 158 | r.trim() 159 | } 160 | return textRows 161 | } 162 | -------------------------------------------------------------------------------- /internal/stat/ttest.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "math" 5 | ) 6 | 7 | // A TTestResult is the result of a t-test. 8 | type TTestResult struct { 9 | // N1 and N2 are the sizes of the input samples. For a 10 | // one-sample t-test, N2 is 0. 11 | N1, N2 int 12 | 13 | // T is the value of the t-statistic for this t-test. 14 | T float64 15 | 16 | // DoF is the degrees of freedom for this t-test. 17 | DoF float64 18 | 19 | // AltHypothesis specifies the alternative hypothesis tested 20 | // by this test against the null hypothesis that there is no 21 | // difference in the means of the samples. 22 | AltHypothesis LocationHypothesis 23 | 24 | // P is p-value for this t-test for the given null hypothesis. 25 | P float64 26 | } 27 | 28 | func newTTestResult(n1, n2 int, t, dof float64, alt LocationHypothesis) *TTestResult { 29 | dist := TDist{dof} 30 | var p float64 31 | switch alt { 32 | case LocationDiffers: 33 | p = 2 * (1 - dist.CDF(math.Abs(t))) 34 | case LocationLess: 35 | p = dist.CDF(t) 36 | case LocationGreater: 37 | p = 1 - dist.CDF(t) 38 | } 39 | return &TTestResult{N1: n1, N2: n2, T: t, DoF: dof, AltHypothesis: alt, P: p} 40 | } 41 | 42 | // A TTestSample is a sample that can be used for a one or two sample 43 | // t-test. 44 | type TTestSample interface { 45 | Weight() float64 46 | Mean() float64 47 | Variance() float64 48 | } 49 | 50 | var () 51 | 52 | // TwoSampleTTest performs a two-sample (unpaired) Student's t-test on 53 | // samples x1 and x2. This is a test of the null hypothesis that x1 54 | // and x2 are drawn from populations with equal means. It assumes x1 55 | // and x2 are independent samples, that the distributions have equal 56 | // variance, and that the populations are normally distributed. 57 | func TwoSampleTTest(x1, x2 TTestSample, alt LocationHypothesis) (*TTestResult, error) { 58 | n1, n2 := x1.Weight(), x2.Weight() 59 | if n1 == 0 || n2 == 0 { 60 | return nil, ErrSampleSize 61 | } 62 | v1, v2 := x1.Variance(), x2.Variance() 63 | if v1 == 0 && v2 == 0 { 64 | return nil, ErrZeroVariance 65 | } 66 | 67 | dof := n1 + n2 - 2 68 | v12 := ((n1-1)*v1 + (n2-1)*v2) / dof 69 | t := (x1.Mean() - x2.Mean()) / math.Sqrt(v12*(1/n1+1/n2)) 70 | return newTTestResult(int(n1), int(n2), t, dof, alt), nil 71 | } 72 | 73 | // TwoSampleWelchTTest performs a two-sample (unpaired) Welch's t-test 74 | // on samples x1 and x2. This is like TwoSampleTTest, but does not 75 | // assume the distributions have equal variance. 76 | func TwoSampleWelchTTest(x1, x2 TTestSample, alt LocationHypothesis) (*TTestResult, error) { 77 | n1, n2 := x1.Weight(), x2.Weight() 78 | if n1 <= 1 || n2 <= 1 { 79 | // TODO: Can we still do this with n == 1? 80 | return nil, ErrSampleSize 81 | } 82 | v1, v2 := x1.Variance(), x2.Variance() 83 | if v1 == 0 && v2 == 0 { 84 | return nil, ErrZeroVariance 85 | } 86 | 87 | dof := math.Pow(v1/n1+v2/n2, 2) / 88 | (math.Pow(v1/n1, 2)/(n1-1) + math.Pow(v2/n2, 2)/(n2-1)) 89 | s := math.Sqrt(v1/n1 + v2/n2) 90 | t := (x1.Mean() - x2.Mean()) / s 91 | return newTTestResult(int(n1), int(n2), t, dof, alt), nil 92 | } 93 | 94 | // PairedTTest performs a two-sample paired t-test on samples x1 and 95 | // x2. If μ0 is non-zero, this tests if the average of the difference 96 | // is significantly different from μ0. If x1 and x2 are identical, 97 | // this returns nil. 98 | func PairedTTest(x1, x2 []float64, μ0 float64, alt LocationHypothesis) (*TTestResult, error) { 99 | if len(x1) != len(x2) { 100 | return nil, ErrMismatchedSamples 101 | } 102 | if len(x1) <= 1 { 103 | // TODO: Can we still do this with n == 1? 104 | return nil, ErrSampleSize 105 | } 106 | 107 | dof := float64(len(x1) - 1) 108 | 109 | diff := make([]float64, len(x1)) 110 | for i := range x1 { 111 | diff[i] = x1[i] - x2[i] 112 | } 113 | sd := StdDev(diff) 114 | if sd == 0 { 115 | // TODO: Can we still do the test? 116 | return nil, ErrZeroVariance 117 | } 118 | t := (Mean(diff) - μ0) * math.Sqrt(float64(len(x1))) / sd 119 | return newTTestResult(len(x1), len(x2), t, dof, alt), nil 120 | } 121 | 122 | // OneSampleTTest performs a one-sample t-test on sample x. This tests 123 | // the null hypothesis that the population mean is equal to μ0. This 124 | // assumes the distribution of the population of sample means is 125 | // normal. 126 | func OneSampleTTest(x TTestSample, μ0 float64, alt LocationHypothesis) (*TTestResult, error) { 127 | n, v := x.Weight(), x.Variance() 128 | if n == 0 { 129 | return nil, ErrSampleSize 130 | } 131 | if v == 0 { 132 | // TODO: Can we still do the test? 133 | return nil, ErrZeroVariance 134 | } 135 | dof := n - 1 136 | t := (x.Mean() - μ0) * math.Sqrt(n) / math.Sqrt(v) 137 | return newTTestResult(int(n), 0, t, dof, alt), nil 138 | } 139 | -------------------------------------------------------------------------------- /internal/stat/udist.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import "math" 4 | 5 | // A UDist is the discrete probability distribution of the 6 | // Mann-Whitney U statistic for a pair of samples of sizes N1 and N2. 7 | // 8 | // The details of computing this distribution with no ties can be 9 | // found in Mann, Henry B.; Whitney, Donald R. (1947). "On a Test of 10 | // Whether one of Two Random Variables is Stochastically Larger than 11 | // the Other". Annals of Mathematical Statistics 18 (1): 50–60. 12 | // Computing this distribution in the presence of ties is described in 13 | // Klotz, J. H. (1966). "The Wilcoxon, Ties, and the Computer". 14 | // Journal of the American Statistical Association 61 (315): 772-787 15 | // and Cheung, Ying Kuen; Klotz, Jerome H. (1997). "The Mann Whitney 16 | // Wilcoxon Distribution Using Linked Lists". Statistica Sinica 7: 17 | // 805-813 (the former paper contains details that are glossed over in 18 | // the latter paper but has mathematical typesetting issues, so it's 19 | // easiest to get the context from the former paper and the details 20 | // from the latter). 21 | type UDist struct { 22 | N1, N2 int 23 | 24 | // T is the count of the number of ties at each rank in the 25 | // input distributions. T may be nil, in which case it is 26 | // assumed there are no ties (which is equivalent to an M+N 27 | // slice of 1s). It must be the case that Sum(T) == M+N. 28 | T []int 29 | } 30 | 31 | // hasTies returns true if d has any tied samples. 32 | func (d UDist) hasTies() bool { 33 | for _, t := range d.T { 34 | if t > 1 { 35 | return true 36 | } 37 | } 38 | return false 39 | } 40 | 41 | // p returns the p_{d.N1,d.N2} function defined by Mann, Whitney 1947 42 | // for values of U from 0 up to and including the U argument. 43 | // 44 | // This algorithm runs in Θ(N1*N2*U) = O(N1²N2²) time and is quite 45 | // fast for small values of N1 and N2. However, it does not handle ties. 46 | func (d UDist) p(U int) []float64 { 47 | // This is a dynamic programming implementation of the 48 | // recursive recurrence definition given by Mann and Whitney: 49 | // 50 | // p_{n,m}(U) = (n * p_{n-1,m}(U-m) + m * p_{n,m-1}(U)) / (n+m) 51 | // p_{n,m}(U) = 0 if U < 0 52 | // p_{0,m}(U) = p{n,0}(U) = 1 / nCr(m+n, n) if U = 0 53 | // = 0 if U > 0 54 | // 55 | // (Note that there is a typo in the original paper. The first 56 | // recursive application of p should be for U-m, not U-M.) 57 | // 58 | // Since p{n,m} only depends on p{n-1,m} and p{n,m-1}, we only 59 | // need to store one "plane" of the three dimensional space at 60 | // a time. 61 | // 62 | // Furthermore, p_{n,m} = p_{m,n}, so we only construct values 63 | // for n <= m and obtain the rest through symmetry. 64 | // 65 | // We organize the computed values of p as followed: 66 | // 67 | // n → N 68 | // m * 69 | // ↓ * * 70 | // * * * 71 | // * * * * 72 | // * * * * 73 | // M * * * * 74 | // 75 | // where each * is a slice indexed by U. The code below 76 | // computes these left-to-right, top-to-bottom, so it only 77 | // stores one row of this matrix at a time. Furthermore, 78 | // computing an element in a given U slice only depends on the 79 | // same and smaller values of U, so we can overwrite the U 80 | // slice we're computing in place as long as we start with the 81 | // largest value of U. Finally, even though the recurrence 82 | // depends on (n,m) above the diagonal and we use symmetry to 83 | // mirror those across the diagonal to (m,n), the mirrored 84 | // indexes are always available in the current row, so this 85 | // mirroring does not interfere with our ability to recycle 86 | // state. 87 | 88 | N, M := d.N1, d.N2 89 | if N > M { 90 | N, M = M, N 91 | } 92 | 93 | memo := make([][]float64, N+1) 94 | for n := range memo { 95 | memo[n] = make([]float64, U+1) 96 | } 97 | 98 | for m := 0; m <= M; m++ { 99 | // Compute p_{0,m}. This is zero except for U=0. 100 | memo[0][0] = 1 101 | 102 | // Compute the remainder of this row. 103 | nlim := N 104 | if m < nlim { 105 | nlim = m 106 | } 107 | for n := 1; n <= nlim; n++ { 108 | lp := memo[n-1] // p_{n-1,m} 109 | var rp []float64 110 | if n <= m-1 { 111 | rp = memo[n] // p_{n,m-1} 112 | } else { 113 | rp = memo[m-1] // p{m-1,n} and m==n 114 | } 115 | 116 | // For a given n,m, U is at most n*m. 117 | // 118 | // TODO: Actually, it's at most ⌈n*m/2⌉, but 119 | // then we need to use more complex symmetries 120 | // in the inner loop below. 121 | ulim := n * m 122 | if U < ulim { 123 | ulim = U 124 | } 125 | 126 | out := memo[n] // p_{n,m} 127 | nplusm := float64(n + m) 128 | for U1 := ulim; U1 >= 0; U1-- { 129 | l := 0.0 130 | if U1-m >= 0 { 131 | l = float64(n) * lp[U1-m] 132 | } 133 | r := float64(m) * rp[U1] 134 | out[U1] = (l + r) / nplusm 135 | } 136 | } 137 | } 138 | return memo[N] 139 | } 140 | 141 | type ukey struct { 142 | n1 int // size of first sample 143 | twoU int // 2*U statistic for this permutation 144 | } 145 | 146 | // This computes the cumulative counts of the Mann-Whitney U 147 | // distribution in the presence of ties using the computation from 148 | // Cheung, Ying Kuen; Klotz, Jerome H. (1997). "The Mann Whitney 149 | // Wilcoxon Distribution Using Linked Lists". Statistica Sinica 7: 150 | // 805-813, with much guidance from appendix L of Klotz, A 151 | // Computational Approach to Statistics. 152 | // 153 | // makeUmemo constructs a table memo[K][ukey{n1, 2*U}], where K is the 154 | // number of ranks (up to len(t)), n1 is the size of the first sample 155 | // (up to the n1 argument), and U is the U statistic (up to the 156 | // argument twoU/2). The value of an entry in the memo table is the 157 | // number of permutations of a sample of size n1 in a ranking with tie 158 | // vector t[:K] having a U statistic <= U. 159 | func makeUmemo(twoU, n1 int, t []int) []map[ukey]float64 { 160 | // Another candidate for a fast implementation is van de Wiel, 161 | // "The split-up algorithm: a fast symbolic method for 162 | // computing p-values of distribution-free statistics". This 163 | // is what's used by R's coin package. It's a comparatively 164 | // recent publication, so it's presumably faster (or perhaps 165 | // just more general) than previous techniques, but I can't 166 | // get my hands on the paper. 167 | // 168 | // TODO: ~40% of this function's time is spent in mapassign on 169 | // the assignment lines in the two loops and another ~20% in 170 | // map access and iteration. Improving map behavior or 171 | // replacing the maps altogether with some other constant-time 172 | // structure could double performance. 173 | // 174 | // TODO: The worst case for this function is when there are 175 | // few ties. Yet the best case overall is when there are *no* 176 | // ties. Can we get the best of both worlds? Use the fast 177 | // algorithm for the most part when there are few ties and mix 178 | // in the general algorithm just where we need it? That's 179 | // certainly possible for sub-problems where t[:k] has no 180 | // ties, but that doesn't help if t[0] has a tie but nothing 181 | // else does. Is it possible to rearrange the ranks without 182 | // messing up our computation of the U statistic for 183 | // sub-problems? 184 | 185 | K := len(t) 186 | 187 | // Compute a coefficients. The a slice is indexed by k (a[0] 188 | // is unused). 189 | a := make([]int, K+1) 190 | a[1] = t[0] 191 | for k := 2; k <= K; k++ { 192 | a[k] = a[k-1] + t[k-2] + t[k-1] 193 | } 194 | 195 | // Create the memo table for the counts function, A. The A 196 | // slice is indexed by k (A[0] is unused). 197 | // 198 | // In "The Mann Whitney Distribution Using Linked Lists", they 199 | // use linked lists (*gasp*) for this, but within each K it's 200 | // really just a memoization table, so it's faster to use a 201 | // map. The outer structure is a slice indexed by k because we 202 | // need to find all memo entries with certain values of k. 203 | // 204 | // TODO: The n1 and twoU values in the ukeys follow strict 205 | // patterns. For each K value, the n1 values are every integer 206 | // between two bounds. For each (K, n1) value, the twoU values 207 | // are every integer multiple of a certain base between two 208 | // bounds. It might be worth turning these into directly 209 | // indexible slices. 210 | A := make([]map[ukey]float64, K+1) 211 | A[K] = map[ukey]float64{ukey{n1: n1, twoU: twoU}: 0} 212 | 213 | // Compute memo table (k, n1, twoU) triples from high K values 214 | // to low K values. This drives the recurrence relation 215 | // downward to figure out all of the needed argument triples. 216 | // 217 | // TODO: Is it possible to generate this table bottom-up? If 218 | // so, this could be a pure dynamic programming algorithm and 219 | // we could discard the K dimension. We could at least store 220 | // the inputs in a more compact representation that replaces 221 | // the twoU dimension with an interval and a step size (as 222 | // suggested by Cheung, Klotz, not that they make it at all 223 | // clear *why* they're suggesting this). 224 | tsum := sumint(t) // always ∑ t[0:k] 225 | for k := K - 1; k >= 2; k-- { 226 | tsum -= t[k] 227 | A[k] = make(map[ukey]float64) 228 | 229 | // Construct A[k] from A[k+1]. 230 | for A_kplus1 := range A[k+1] { 231 | rkLow := maxint(0, A_kplus1.n1-tsum) 232 | rkHigh := minint(A_kplus1.n1, t[k]) 233 | for rk := rkLow; rk <= rkHigh; rk++ { 234 | twoU_k := A_kplus1.twoU - rk*(a[k+1]-2*A_kplus1.n1+rk) 235 | n1_k := A_kplus1.n1 - rk 236 | if twoUmin(n1_k, t[:k], a) <= twoU_k && twoU_k <= twoUmax(n1_k, t[:k], a) { 237 | key := ukey{n1: n1_k, twoU: twoU_k} 238 | A[k][key] = 0 239 | } 240 | } 241 | } 242 | } 243 | 244 | // Fill counts in memo table from low K values to high K 245 | // values. This unwinds the recurrence relation. 246 | 247 | // Start with K==2 base case. 248 | // 249 | // TODO: Later computations depend on these, but these don't 250 | // depend on anything (including each other), so if K==2, we 251 | // can skip the memo table altogether. 252 | if K < 2 { 253 | panic("K < 2") 254 | } 255 | N_2 := t[0] + t[1] 256 | for A_2i := range A[2] { 257 | Asum := 0.0 258 | r2Low := maxint(0, A_2i.n1-t[0]) 259 | r2High := (A_2i.twoU - A_2i.n1*(t[0]-A_2i.n1)) / N_2 260 | for r2 := r2Low; r2 <= r2High; r2++ { 261 | Asum += mathChoose(t[0], A_2i.n1-r2) * 262 | mathChoose(t[1], r2) 263 | } 264 | A[2][A_2i] = Asum 265 | } 266 | 267 | // Derive counts for the rest of the memo table. 268 | tsum = t[0] // always ∑ t[0:k-1] 269 | for k := 3; k <= K; k++ { 270 | tsum += t[k-2] 271 | 272 | // Compute A[k] counts from A[k-1] counts. 273 | for A_ki := range A[k] { 274 | Asum := 0.0 275 | rkLow := maxint(0, A_ki.n1-tsum) 276 | rkHigh := minint(A_ki.n1, t[k-1]) 277 | for rk := rkLow; rk <= rkHigh; rk++ { 278 | twoU_kminus1 := A_ki.twoU - rk*(a[k]-2*A_ki.n1+rk) 279 | n1_kminus1 := A_ki.n1 - rk 280 | x, ok := A[k-1][ukey{n1: n1_kminus1, twoU: twoU_kminus1}] 281 | if !ok && twoUmax(n1_kminus1, t[:k-1], a) < twoU_kminus1 { 282 | x = mathChoose(tsum, n1_kminus1) 283 | } 284 | Asum += x * mathChoose(t[k-1], rk) 285 | } 286 | A[k][A_ki] = Asum 287 | } 288 | } 289 | 290 | return A 291 | } 292 | 293 | func twoUmin(n1 int, t, a []int) int { 294 | K := len(t) 295 | twoU := -n1 * n1 296 | n1_k := n1 297 | for k := 1; k <= K; k++ { 298 | twoU_k := minint(n1_k, t[k-1]) 299 | twoU += twoU_k * a[k] 300 | n1_k -= twoU_k 301 | } 302 | return twoU 303 | } 304 | 305 | func twoUmax(n1 int, t, a []int) int { 306 | K := len(t) 307 | twoU := -n1 * n1 308 | n1_k := n1 309 | for k := K; k > 0; k-- { 310 | twoU_k := minint(n1_k, t[k-1]) 311 | twoU += twoU_k * a[k] 312 | n1_k -= twoU_k 313 | } 314 | return twoU 315 | } 316 | 317 | func (d UDist) PMF(U float64) float64 { 318 | if U < 0 || U >= 0.5+float64(d.N1*d.N2) { 319 | return 0 320 | } 321 | 322 | if d.hasTies() { 323 | // makeUmemo computes the CDF directly. Take its 324 | // difference to get the PMF. 325 | p1, ok1 := makeUmemo(int(2*U)-1, d.N1, d.T)[len(d.T)][ukey{d.N1, int(2*U) - 1}] 326 | p2, ok2 := makeUmemo(int(2*U), d.N1, d.T)[len(d.T)][ukey{d.N1, int(2 * U)}] 327 | if !ok1 || !ok2 { 328 | panic("makeUmemo did not return expected memoization table") 329 | } 330 | return (p2 - p1) / mathChoose(d.N1+d.N2, d.N1) 331 | } 332 | 333 | // There are no ties. Use the fast algorithm. U must be integral. 334 | Ui := int(math.Floor(U)) 335 | // TODO: Use symmetry to minimize U 336 | return d.p(Ui)[Ui] 337 | } 338 | 339 | func (d UDist) CDF(U float64) float64 { 340 | if U < 0 { 341 | return 0 342 | } else if U >= float64(d.N1*d.N2) { 343 | return 1 344 | } 345 | 346 | if d.hasTies() { 347 | // TODO: Minimize U? 348 | p, ok := makeUmemo(int(2*U), d.N1, d.T)[len(d.T)][ukey{d.N1, int(2 * U)}] 349 | if !ok { 350 | panic("makeUmemo did not return expected memoization table") 351 | } 352 | return p / mathChoose(d.N1+d.N2, d.N1) 353 | } 354 | 355 | // There are no ties. Use the fast algorithm. U must be integral. 356 | Ui := int(math.Floor(U)) 357 | // The distribution is symmetric around U = m * n / 2. Sum up 358 | // whichever tail is smaller. 359 | flip := Ui >= (d.N1*d.N2+1)/2 360 | if flip { 361 | Ui = d.N1*d.N2 - Ui - 1 362 | } 363 | pdfs := d.p(Ui) 364 | p := 0.0 365 | for _, pdf := range pdfs[:Ui+1] { 366 | p += pdf 367 | } 368 | if flip { 369 | p = 1 - p 370 | } 371 | return p 372 | } 373 | 374 | func (d UDist) Step() float64 { 375 | return 0.5 376 | } 377 | 378 | func (d UDist) Bounds() (float64, float64) { 379 | // TODO: More precise bounds when there are ties. 380 | return 0, float64(d.N1 * d.N2) 381 | } 382 | -------------------------------------------------------------------------------- /internal/stat/utest.go: -------------------------------------------------------------------------------- 1 | package stat 2 | 3 | import ( 4 | "math" 5 | "sort" 6 | ) 7 | 8 | // A LocationHypothesis specifies the alternative hypothesis of a 9 | // location test such as a t-test or a Mann-Whitney U-test. The 10 | // default (zero) value is to test against the alternative hypothesis 11 | // that they differ. 12 | type LocationHypothesis int 13 | 14 | //go:generate stringer -type LocationHypothesis 15 | 16 | const ( 17 | // LocationLess specifies the alternative hypothesis that the 18 | // location of the first sample is less than the second. This 19 | // is a one-tailed test. 20 | LocationLess LocationHypothesis = -1 21 | 22 | // LocationDiffers specifies the alternative hypothesis that 23 | // the locations of the two samples are not equal. This is a 24 | // two-tailed test. 25 | LocationDiffers LocationHypothesis = 0 26 | 27 | // LocationGreater specifies the alternative hypothesis that 28 | // the location of the first sample is greater than the 29 | // second. This is a one-tailed test. 30 | LocationGreater LocationHypothesis = 1 31 | ) 32 | 33 | // A MannWhitneyUTestResult is the result of a Mann-Whitney U-test. 34 | type MannWhitneyUTestResult struct { 35 | // N1 and N2 are the sizes of the input samples. 36 | N1, N2 int 37 | 38 | // U is the value of the Mann-Whitney U statistic for this 39 | // test, generalized by counting ties as 0.5. 40 | // 41 | // Given the Cartesian product of the two samples, this is the 42 | // number of pairs in which the value from the first sample is 43 | // greater than the value of the second, plus 0.5 times the 44 | // number of pairs where the values from the two samples are 45 | // equal. Hence, U is always an integer multiple of 0.5 (it is 46 | // a whole integer if there are no ties) in the range [0, N1*N2]. 47 | // 48 | // U statistics always come in pairs, depending on which 49 | // sample is "first". The mirror U for the other sample can be 50 | // calculated as N1*N2 - U. 51 | // 52 | // There are many equivalent statistics with slightly 53 | // different definitions. The Wilcoxon (1945) W statistic 54 | // (generalized for ties) is U + (N1(N1+1))/2. It is also 55 | // common to use 2U to eliminate the half steps and Smid 56 | // (1956) uses N1*N2 - 2U to additionally center the 57 | // distribution. 58 | U float64 59 | 60 | // AltHypothesis specifies the alternative hypothesis tested 61 | // by this test against the null hypothesis that there is no 62 | // difference in the locations of the samples. 63 | AltHypothesis LocationHypothesis 64 | 65 | // P is the p-value of the Mann-Whitney test for the given 66 | // null hypothesis. 67 | P float64 68 | } 69 | 70 | // MannWhitneyExactLimit gives the largest sample size for which the 71 | // exact U distribution will be used for the Mann-Whitney U-test. 72 | // 73 | // Using the exact distribution is necessary for small sample sizes 74 | // because the distribution is highly irregular. However, computing 75 | // the distribution for large sample sizes is both computationally 76 | // expensive and unnecessary because it quickly approaches a normal 77 | // approximation. Computing the distribution for two 50 value samples 78 | // takes a few milliseconds on a 2014 laptop. 79 | var MannWhitneyExactLimit = 50 80 | 81 | // MannWhitneyTiesExactLimit gives the largest sample size for which 82 | // the exact U distribution will be used for the Mann-Whitney U-test 83 | // in the presence of ties. 84 | // 85 | // Computing this distribution is more expensive than computing the 86 | // distribution without ties, so this is set lower. Computing this 87 | // distribution for two 25 value samples takes about ten milliseconds 88 | // on a 2014 laptop. 89 | var MannWhitneyTiesExactLimit = 25 90 | 91 | // MannWhitneyUTest performs a Mann-Whitney U-test [1,2] of the null 92 | // hypothesis that two samples come from the same population against 93 | // the alternative hypothesis that one sample tends to have larger or 94 | // smaller values than the other. 95 | // 96 | // This is similar to a t-test, but unlike the t-test, the 97 | // Mann-Whitney U-test is non-parametric (it does not assume a normal 98 | // distribution). It has very slightly lower efficiency than the 99 | // t-test on normal distributions. 100 | // 101 | // Computing the exact U distribution is expensive for large sample 102 | // sizes, so this uses a normal approximation for sample sizes larger 103 | // than MannWhitneyExactLimit if there are no ties or 104 | // MannWhitneyTiesExactLimit if there are ties. This normal 105 | // approximation uses both the tie correction and the continuity 106 | // correction. 107 | // 108 | // This can fail with ErrSampleSize if either sample is empty or 109 | // ErrSamplesEqual if all sample values are equal. 110 | // 111 | // This is also known as a Mann-Whitney-Wilcoxon test and is 112 | // equivalent to the Wilcoxon rank-sum test, though the Wilcoxon 113 | // rank-sum test differs in nomenclature. 114 | // 115 | // [1] Mann, Henry B.; Whitney, Donald R. (1947). "On a Test of 116 | // Whether one of Two Random Variables is Stochastically Larger than 117 | // the Other". Annals of Mathematical Statistics 18 (1): 50–60. 118 | // 119 | // [2] Klotz, J. H. (1966). "The Wilcoxon, Ties, and the Computer". 120 | // Journal of the American Statistical Association 61 (315): 772-787. 121 | func MannWhitneyUTest(x1, x2 []float64, alt LocationHypothesis) (*MannWhitneyUTestResult, error) { 122 | n1, n2 := len(x1), len(x2) 123 | if n1 == 0 || n2 == 0 { 124 | return nil, ErrSampleSize 125 | } 126 | 127 | // Compute the U statistic and tie vector T. 128 | x1 = append([]float64(nil), x1...) 129 | x2 = append([]float64(nil), x2...) 130 | sort.Float64s(x1) 131 | sort.Float64s(x2) 132 | merged, labels := labeledMerge(x1, x2) 133 | 134 | R1 := 0.0 135 | T, hasTies := []int{}, false 136 | for i := 0; i < len(merged); { 137 | rank1, nx1, v1 := i+1, 0, merged[i] 138 | // Consume samples that tie this sample (including itself). 139 | for ; i < len(merged) && merged[i] == v1; i++ { 140 | if labels[i] == 1 { 141 | nx1++ 142 | } 143 | } 144 | // Assign all tied samples the average rank of the 145 | // samples, where merged[0] has rank 1. 146 | if nx1 != 0 { 147 | rank := float64(i+rank1) / 2 148 | R1 += rank * float64(nx1) 149 | } 150 | T = append(T, i-rank1+1) 151 | if i > rank1 { 152 | hasTies = true 153 | } 154 | } 155 | U1 := R1 - float64(n1*(n1+1))/2 156 | 157 | // Compute the smaller of U1 and U2 158 | U2 := float64(n1*n2) - U1 159 | Usmall := math.Min(U1, U2) 160 | 161 | var p float64 162 | if !hasTies && n1 <= MannWhitneyExactLimit && n2 <= MannWhitneyExactLimit || 163 | hasTies && n1 <= MannWhitneyTiesExactLimit && n2 <= MannWhitneyTiesExactLimit { 164 | // Use exact U distribution. U1 will be an integer. 165 | if len(T) == 1 { 166 | // All values are equal. Test is meaningless. 167 | return nil, ErrSamplesEqual 168 | } 169 | 170 | dist := UDist{N1: n1, N2: n2, T: T} 171 | switch alt { 172 | case LocationDiffers: 173 | if U1 == U2 { 174 | // The distribution is symmetric about 175 | // Usmall. Since the distribution is 176 | // discrete, the CDF is discontinuous 177 | // and if simply double CDF(Usmall), 178 | // we'll double count the 179 | // (non-infinitesimal) probability 180 | // mass at Usmall. What we want is 181 | // just the integral of the whole CDF, 182 | // which is 1. 183 | p = 1 184 | } else { 185 | p = dist.CDF(Usmall) * 2 186 | } 187 | 188 | case LocationLess: 189 | p = dist.CDF(U1) 190 | 191 | case LocationGreater: 192 | p = 1 - dist.CDF(U1-1) 193 | } 194 | } else { 195 | // Use normal approximation (with tie and continuity 196 | // correction). 197 | t := tieCorrection(T) 198 | N := float64(n1 + n2) 199 | μU := float64(n1*n2) / 2 200 | σU := math.Sqrt(float64(n1*n2) * ((N + 1) - t/(N*(N-1))) / 12) 201 | if σU == 0 { 202 | return nil, ErrSamplesEqual 203 | } 204 | numer := U1 - μU 205 | // Perform continuity correction. 206 | switch alt { 207 | case LocationDiffers: 208 | numer -= mathSign(numer) * 0.5 209 | case LocationLess: 210 | numer += 0.5 211 | case LocationGreater: 212 | numer -= 0.5 213 | } 214 | z := numer / σU 215 | switch alt { 216 | case LocationDiffers: 217 | p = 2 * math.Min(StdNormal.CDF(z), 1-StdNormal.CDF(z)) 218 | case LocationLess: 219 | p = StdNormal.CDF(z) 220 | case LocationGreater: 221 | p = 1 - StdNormal.CDF(z) 222 | } 223 | } 224 | 225 | return &MannWhitneyUTestResult{N1: n1, N2: n2, U: U1, 226 | AltHypothesis: alt, P: p}, nil 227 | } 228 | 229 | // labeledMerge merges sorted lists x1 and x2 into sorted list merged. 230 | // labels[i] is 1 or 2 depending on whether merged[i] is a value from 231 | // x1 or x2, respectively. 232 | func labeledMerge(x1, x2 []float64) (merged []float64, labels []byte) { 233 | merged = make([]float64, len(x1)+len(x2)) 234 | labels = make([]byte, len(x1)+len(x2)) 235 | 236 | i, j, o := 0, 0, 0 237 | for i < len(x1) && j < len(x2) { 238 | if x1[i] < x2[j] { 239 | merged[o] = x1[i] 240 | labels[o] = 1 241 | i++ 242 | } else { 243 | merged[o] = x2[j] 244 | labels[o] = 2 245 | j++ 246 | } 247 | o++ 248 | } 249 | for ; i < len(x1); i++ { 250 | merged[o] = x1[i] 251 | labels[o] = 1 252 | o++ 253 | } 254 | for ; j < len(x2); j++ { 255 | merged[o] = x2[j] 256 | labels[o] = 2 257 | o++ 258 | } 259 | return 260 | } 261 | 262 | // tieCorrection computes the tie correction factor Σ_j (t_j³ - t_j) 263 | // where t_j is the number of ties in the j'th rank. 264 | func tieCorrection(ties []int) float64 { 265 | t := 0 266 | for _, tie := range ties { 267 | t += tie*tie*tie - tie 268 | } 269 | return float64(t) 270 | } 271 | -------------------------------------------------------------------------------- /internal/term/color.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The golang.design Initiative Authors. 2 | // All rights reserved. Use of this source code is governed 3 | // by a GNU GPLv3 license that can be found in the LICENSE file. 4 | 5 | package term 6 | 7 | // Red turns a given string to red 8 | func Red(in string) string { 9 | return fgString(in, 255, 0, 0) 10 | } 11 | 12 | // Orange turns a given string to orange 13 | func Orange(in string) string { 14 | return fgString(in, 252, 140, 3) 15 | } 16 | 17 | // Green turns a given string to green 18 | func Green(in string) string { 19 | return fgString(in, 0, 255, 0) 20 | } 21 | 22 | // Gray turns a given string to gray 23 | func Gray(in string) string { 24 | return fgString(in, 125, 125, 125) 25 | } 26 | 27 | var ( 28 | before = []byte("\033[") 29 | after = []byte("m") 30 | reset = []byte("\033[0;00m") 31 | fgcolors = fgTermRGB[16:232] 32 | bgcolors = bgTermRGB[16:232] 33 | ) 34 | 35 | func fgString(in string, r, g, b uint8) string { 36 | return string(fgBytes([]byte(in), r, g, b)) 37 | } 38 | 39 | // Bytes colorizes the foreground with the terminal color that matches 40 | // the closest the RGB color. 41 | func fgBytes(in []byte, r, g, b uint8) []byte { 42 | return colorize(color(r, g, b, true), in) 43 | } 44 | 45 | func colorize(color, in []byte) []byte { 46 | return append(append(append(append(before, color...), after...), in...), reset...) 47 | } 48 | 49 | func color(r, g, b uint8, foreground bool) []byte { 50 | // if all colors are equal, it might be in the grayscale range 51 | if r == g && g == b { 52 | color, ok := grayscale(r, foreground) 53 | if ok { 54 | return color 55 | } 56 | } 57 | 58 | // the general case approximates RGB by using the closest color. 59 | r6 := ((uint16(r) * 5) / 255) 60 | g6 := ((uint16(g) * 5) / 255) 61 | b6 := ((uint16(b) * 5) / 255) 62 | i := 36*r6 + 6*g6 + b6 63 | if foreground { 64 | return fgcolors[i] 65 | } 66 | return bgcolors[i] 67 | } 68 | 69 | func grayscale(scale uint8, foreground bool) ([]byte, bool) { 70 | var source [256][]byte 71 | 72 | if foreground { 73 | source = fgTermRGB 74 | } else { 75 | source = bgTermRGB 76 | } 77 | 78 | switch scale { 79 | case 0x08: 80 | return source[232], true 81 | case 0x12: 82 | return source[233], true 83 | case 0x1c: 84 | return source[234], true 85 | case 0x26: 86 | return source[235], true 87 | case 0x30: 88 | return source[236], true 89 | case 0x3a: 90 | return source[237], true 91 | case 0x44: 92 | return source[238], true 93 | case 0x4e: 94 | return source[239], true 95 | case 0x58: 96 | return source[240], true 97 | case 0x62: 98 | return source[241], true 99 | case 0x6c: 100 | return source[242], true 101 | case 0x76: 102 | return source[243], true 103 | case 0x80: 104 | return source[244], true 105 | case 0x8a: 106 | return source[245], true 107 | case 0x94: 108 | return source[246], true 109 | case 0x9e: 110 | return source[247], true 111 | case 0xa8: 112 | return source[248], true 113 | case 0xb2: 114 | return source[249], true 115 | case 0xbc: 116 | return source[250], true 117 | case 0xc6: 118 | return source[251], true 119 | case 0xd0: 120 | return source[252], true 121 | case 0xda: 122 | return source[253], true 123 | case 0xe4: 124 | return source[254], true 125 | case 0xee: 126 | return source[255], true 127 | } 128 | return nil, false 129 | } 130 | 131 | var ( 132 | yellow = fgString("", 252, 140, 3) 133 | red = fgString("", 255, 0, 0) 134 | green = fgString("", 0, 255, 0) 135 | ) 136 | 137 | // \033[ 138 | 139 | var fgTermRGB = [...][]byte{ 140 | []byte("38;5;0"), 141 | []byte("38;5;1"), 142 | []byte("38;5;2"), 143 | []byte("38;5;3"), 144 | []byte("38;5;4"), 145 | []byte("38;5;5"), 146 | []byte("38;5;6"), 147 | []byte("38;5;7"), 148 | []byte("38;5;8"), 149 | []byte("38;5;9"), 150 | []byte("38;5;10"), 151 | []byte("38;5;11"), 152 | []byte("38;5;12"), 153 | []byte("38;5;13"), 154 | []byte("38;5;14"), 155 | []byte("38;5;15"), 156 | []byte("38;5;16"), 157 | []byte("38;5;17"), 158 | []byte("38;5;18"), 159 | []byte("38;5;19"), 160 | []byte("38;5;20"), 161 | []byte("38;5;21"), 162 | []byte("38;5;22"), 163 | []byte("38;5;23"), 164 | []byte("38;5;24"), 165 | []byte("38;5;25"), 166 | []byte("38;5;26"), 167 | []byte("38;5;27"), 168 | []byte("38;5;28"), 169 | []byte("38;5;29"), 170 | []byte("38;5;30"), 171 | []byte("38;5;31"), 172 | []byte("38;5;32"), 173 | []byte("38;5;33"), 174 | []byte("38;5;34"), 175 | []byte("38;5;35"), 176 | []byte("38;5;36"), 177 | []byte("38;5;37"), 178 | []byte("38;5;38"), 179 | []byte("38;5;39"), 180 | []byte("38;5;40"), 181 | []byte("38;5;41"), 182 | []byte("38;5;42"), 183 | []byte("38;5;43"), 184 | []byte("38;5;44"), 185 | []byte("38;5;45"), 186 | []byte("38;5;46"), 187 | []byte("38;5;47"), 188 | []byte("38;5;48"), 189 | []byte("38;5;49"), 190 | []byte("38;5;50"), 191 | []byte("38;5;51"), 192 | []byte("38;5;52"), 193 | []byte("38;5;53"), 194 | []byte("38;5;54"), 195 | []byte("38;5;55"), 196 | []byte("38;5;56"), 197 | []byte("38;5;57"), 198 | []byte("38;5;58"), 199 | []byte("38;5;59"), 200 | []byte("38;5;60"), 201 | []byte("38;5;61"), 202 | []byte("38;5;62"), 203 | []byte("38;5;63"), 204 | []byte("38;5;64"), 205 | []byte("38;5;65"), 206 | []byte("38;5;66"), 207 | []byte("38;5;67"), 208 | []byte("38;5;68"), 209 | []byte("38;5;69"), 210 | []byte("38;5;70"), 211 | []byte("38;5;71"), 212 | []byte("38;5;72"), 213 | []byte("38;5;73"), 214 | []byte("38;5;74"), 215 | []byte("38;5;75"), 216 | []byte("38;5;76"), 217 | []byte("38;5;77"), 218 | []byte("38;5;78"), 219 | []byte("38;5;79"), 220 | []byte("38;5;80"), 221 | []byte("38;5;81"), 222 | []byte("38;5;82"), 223 | []byte("38;5;83"), 224 | []byte("38;5;84"), 225 | []byte("38;5;85"), 226 | []byte("38;5;86"), 227 | []byte("38;5;87"), 228 | []byte("38;5;88"), 229 | []byte("38;5;89"), 230 | []byte("38;5;90"), 231 | []byte("38;5;91"), 232 | []byte("38;5;92"), 233 | []byte("38;5;93"), 234 | []byte("38;5;94"), 235 | []byte("38;5;95"), 236 | []byte("38;5;96"), 237 | []byte("38;5;97"), 238 | []byte("38;5;98"), 239 | []byte("38;5;99"), 240 | []byte("38;5;100"), 241 | []byte("38;5;101"), 242 | []byte("38;5;102"), 243 | []byte("38;5;103"), 244 | []byte("38;5;104"), 245 | []byte("38;5;105"), 246 | []byte("38;5;106"), 247 | []byte("38;5;107"), 248 | []byte("38;5;108"), 249 | []byte("38;5;109"), 250 | []byte("38;5;110"), 251 | []byte("38;5;111"), 252 | []byte("38;5;112"), 253 | []byte("38;5;113"), 254 | []byte("38;5;114"), 255 | []byte("38;5;115"), 256 | []byte("38;5;116"), 257 | []byte("38;5;117"), 258 | []byte("38;5;118"), 259 | []byte("38;5;119"), 260 | []byte("38;5;120"), 261 | []byte("38;5;121"), 262 | []byte("38;5;122"), 263 | []byte("38;5;123"), 264 | []byte("38;5;124"), 265 | []byte("38;5;125"), 266 | []byte("38;5;126"), 267 | []byte("38;5;127"), 268 | []byte("38;5;128"), 269 | []byte("38;5;129"), 270 | []byte("38;5;130"), 271 | []byte("38;5;131"), 272 | []byte("38;5;132"), 273 | []byte("38;5;133"), 274 | []byte("38;5;134"), 275 | []byte("38;5;135"), 276 | []byte("38;5;136"), 277 | []byte("38;5;137"), 278 | []byte("38;5;138"), 279 | []byte("38;5;139"), 280 | []byte("38;5;140"), 281 | []byte("38;5;141"), 282 | []byte("38;5;142"), 283 | []byte("38;5;143"), 284 | []byte("38;5;144"), 285 | []byte("38;5;145"), 286 | []byte("38;5;146"), 287 | []byte("38;5;147"), 288 | []byte("38;5;148"), 289 | []byte("38;5;149"), 290 | []byte("38;5;150"), 291 | []byte("38;5;151"), 292 | []byte("38;5;152"), 293 | []byte("38;5;153"), 294 | []byte("38;5;154"), 295 | []byte("38;5;155"), 296 | []byte("38;5;156"), 297 | []byte("38;5;157"), 298 | []byte("38;5;158"), 299 | []byte("38;5;159"), 300 | []byte("38;5;160"), 301 | []byte("38;5;161"), 302 | []byte("38;5;162"), 303 | []byte("38;5;163"), 304 | []byte("38;5;164"), 305 | []byte("38;5;165"), 306 | []byte("38;5;166"), 307 | []byte("38;5;167"), 308 | []byte("38;5;168"), 309 | []byte("38;5;169"), 310 | []byte("38;5;170"), 311 | []byte("38;5;171"), 312 | []byte("38;5;172"), 313 | []byte("38;5;173"), 314 | []byte("38;5;174"), 315 | []byte("38;5;175"), 316 | []byte("38;5;176"), 317 | []byte("38;5;177"), 318 | []byte("38;5;178"), 319 | []byte("38;5;179"), 320 | []byte("38;5;180"), 321 | []byte("38;5;181"), 322 | []byte("38;5;182"), 323 | []byte("38;5;183"), 324 | []byte("38;5;184"), 325 | []byte("38;5;185"), 326 | []byte("38;5;186"), 327 | []byte("38;5;187"), 328 | []byte("38;5;188"), 329 | []byte("38;5;189"), 330 | []byte("38;5;190"), 331 | []byte("38;5;191"), 332 | []byte("38;5;192"), 333 | []byte("38;5;193"), 334 | []byte("38;5;194"), 335 | []byte("38;5;195"), 336 | []byte("38;5;196"), 337 | []byte("38;5;197"), 338 | []byte("38;5;198"), 339 | []byte("38;5;199"), 340 | []byte("38;5;200"), 341 | []byte("38;5;201"), 342 | []byte("38;5;202"), 343 | []byte("38;5;203"), 344 | []byte("38;5;204"), 345 | []byte("38;5;205"), 346 | []byte("38;5;206"), 347 | []byte("38;5;207"), 348 | []byte("38;5;208"), 349 | []byte("38;5;209"), 350 | []byte("38;5;210"), 351 | []byte("38;5;211"), 352 | []byte("38;5;212"), 353 | []byte("38;5;213"), 354 | []byte("38;5;214"), 355 | []byte("38;5;215"), 356 | []byte("38;5;216"), 357 | []byte("38;5;217"), 358 | []byte("38;5;218"), 359 | []byte("38;5;219"), 360 | []byte("38;5;220"), 361 | []byte("38;5;221"), 362 | []byte("38;5;222"), 363 | []byte("38;5;223"), 364 | []byte("38;5;224"), 365 | []byte("38;5;225"), 366 | []byte("38;5;226"), 367 | []byte("38;5;227"), 368 | []byte("38;5;228"), 369 | []byte("38;5;229"), 370 | []byte("38;5;230"), 371 | []byte("38;5;231"), 372 | []byte("38;5;232"), 373 | []byte("38;5;233"), 374 | []byte("38;5;234"), 375 | []byte("38;5;235"), 376 | []byte("38;5;236"), 377 | []byte("38;5;237"), 378 | []byte("38;5;238"), 379 | []byte("38;5;239"), 380 | []byte("38;5;240"), 381 | []byte("38;5;241"), 382 | []byte("38;5;242"), 383 | []byte("38;5;243"), 384 | []byte("38;5;244"), 385 | []byte("38;5;245"), 386 | []byte("38;5;246"), 387 | []byte("38;5;247"), 388 | []byte("38;5;248"), 389 | []byte("38;5;249"), 390 | []byte("38;5;250"), 391 | []byte("38;5;251"), 392 | []byte("38;5;252"), 393 | []byte("38;5;253"), 394 | []byte("38;5;254"), 395 | []byte("38;5;255"), 396 | } 397 | 398 | var bgTermRGB = [...][]byte{ 399 | []byte("48;5;0"), 400 | []byte("48;5;1"), 401 | []byte("48;5;2"), 402 | []byte("48;5;3"), 403 | []byte("48;5;4"), 404 | []byte("48;5;5"), 405 | []byte("48;5;6"), 406 | []byte("48;5;7"), 407 | []byte("48;5;8"), 408 | []byte("48;5;9"), 409 | []byte("48;5;10"), 410 | []byte("48;5;11"), 411 | []byte("48;5;12"), 412 | []byte("48;5;13"), 413 | []byte("48;5;14"), 414 | []byte("48;5;15"), 415 | []byte("48;5;16"), 416 | []byte("48;5;17"), 417 | []byte("48;5;18"), 418 | []byte("48;5;19"), 419 | []byte("48;5;20"), 420 | []byte("48;5;21"), 421 | []byte("48;5;22"), 422 | []byte("48;5;23"), 423 | []byte("48;5;24"), 424 | []byte("48;5;25"), 425 | []byte("48;5;26"), 426 | []byte("48;5;27"), 427 | []byte("48;5;28"), 428 | []byte("48;5;29"), 429 | []byte("48;5;30"), 430 | []byte("48;5;31"), 431 | []byte("48;5;32"), 432 | []byte("48;5;33"), 433 | []byte("48;5;34"), 434 | []byte("48;5;35"), 435 | []byte("48;5;36"), 436 | []byte("48;5;37"), 437 | []byte("48;5;38"), 438 | []byte("48;5;39"), 439 | []byte("48;5;40"), 440 | []byte("48;5;41"), 441 | []byte("48;5;42"), 442 | []byte("48;5;43"), 443 | []byte("48;5;44"), 444 | []byte("48;5;45"), 445 | []byte("48;5;46"), 446 | []byte("48;5;47"), 447 | []byte("48;5;48"), 448 | []byte("48;5;49"), 449 | []byte("48;5;50"), 450 | []byte("48;5;51"), 451 | []byte("48;5;52"), 452 | []byte("48;5;53"), 453 | []byte("48;5;54"), 454 | []byte("48;5;55"), 455 | []byte("48;5;56"), 456 | []byte("48;5;57"), 457 | []byte("48;5;58"), 458 | []byte("48;5;59"), 459 | []byte("48;5;60"), 460 | []byte("48;5;61"), 461 | []byte("48;5;62"), 462 | []byte("48;5;63"), 463 | []byte("48;5;64"), 464 | []byte("48;5;65"), 465 | []byte("48;5;66"), 466 | []byte("48;5;67"), 467 | []byte("48;5;68"), 468 | []byte("48;5;69"), 469 | []byte("48;5;70"), 470 | []byte("48;5;71"), 471 | []byte("48;5;72"), 472 | []byte("48;5;73"), 473 | []byte("48;5;74"), 474 | []byte("48;5;75"), 475 | []byte("48;5;76"), 476 | []byte("48;5;77"), 477 | []byte("48;5;78"), 478 | []byte("48;5;79"), 479 | []byte("48;5;80"), 480 | []byte("48;5;81"), 481 | []byte("48;5;82"), 482 | []byte("48;5;83"), 483 | []byte("48;5;84"), 484 | []byte("48;5;85"), 485 | []byte("48;5;86"), 486 | []byte("48;5;87"), 487 | []byte("48;5;88"), 488 | []byte("48;5;89"), 489 | []byte("48;5;90"), 490 | []byte("48;5;91"), 491 | []byte("48;5;92"), 492 | []byte("48;5;93"), 493 | []byte("48;5;94"), 494 | []byte("48;5;95"), 495 | []byte("48;5;96"), 496 | []byte("48;5;97"), 497 | []byte("48;5;98"), 498 | []byte("48;5;99"), 499 | []byte("48;5;100"), 500 | []byte("48;5;101"), 501 | []byte("48;5;102"), 502 | []byte("48;5;103"), 503 | []byte("48;5;104"), 504 | []byte("48;5;105"), 505 | []byte("48;5;106"), 506 | []byte("48;5;107"), 507 | []byte("48;5;108"), 508 | []byte("48;5;109"), 509 | []byte("48;5;110"), 510 | []byte("48;5;111"), 511 | []byte("48;5;112"), 512 | []byte("48;5;113"), 513 | []byte("48;5;114"), 514 | []byte("48;5;115"), 515 | []byte("48;5;116"), 516 | []byte("48;5;117"), 517 | []byte("48;5;118"), 518 | []byte("48;5;119"), 519 | []byte("48;5;120"), 520 | []byte("48;5;121"), 521 | []byte("48;5;122"), 522 | []byte("48;5;123"), 523 | []byte("48;5;124"), 524 | []byte("48;5;125"), 525 | []byte("48;5;126"), 526 | []byte("48;5;127"), 527 | []byte("48;5;128"), 528 | []byte("48;5;129"), 529 | []byte("48;5;130"), 530 | []byte("48;5;131"), 531 | []byte("48;5;132"), 532 | []byte("48;5;133"), 533 | []byte("48;5;134"), 534 | []byte("48;5;135"), 535 | []byte("48;5;136"), 536 | []byte("48;5;137"), 537 | []byte("48;5;138"), 538 | []byte("48;5;139"), 539 | []byte("48;5;140"), 540 | []byte("48;5;141"), 541 | []byte("48;5;142"), 542 | []byte("48;5;143"), 543 | []byte("48;5;144"), 544 | []byte("48;5;145"), 545 | []byte("48;5;146"), 546 | []byte("48;5;147"), 547 | []byte("48;5;148"), 548 | []byte("48;5;149"), 549 | []byte("48;5;150"), 550 | []byte("48;5;151"), 551 | []byte("48;5;152"), 552 | []byte("48;5;153"), 553 | []byte("48;5;154"), 554 | []byte("48;5;155"), 555 | []byte("48;5;156"), 556 | []byte("48;5;157"), 557 | []byte("48;5;158"), 558 | []byte("48;5;159"), 559 | []byte("48;5;160"), 560 | []byte("48;5;161"), 561 | []byte("48;5;162"), 562 | []byte("48;5;163"), 563 | []byte("48;5;164"), 564 | []byte("48;5;165"), 565 | []byte("48;5;166"), 566 | []byte("48;5;167"), 567 | []byte("48;5;168"), 568 | []byte("48;5;169"), 569 | []byte("48;5;170"), 570 | []byte("48;5;171"), 571 | []byte("48;5;172"), 572 | []byte("48;5;173"), 573 | []byte("48;5;174"), 574 | []byte("48;5;175"), 575 | []byte("48;5;176"), 576 | []byte("48;5;177"), 577 | []byte("48;5;178"), 578 | []byte("48;5;179"), 579 | []byte("48;5;180"), 580 | []byte("48;5;181"), 581 | []byte("48;5;182"), 582 | []byte("48;5;183"), 583 | []byte("48;5;184"), 584 | []byte("48;5;185"), 585 | []byte("48;5;186"), 586 | []byte("48;5;187"), 587 | []byte("48;5;188"), 588 | []byte("48;5;189"), 589 | []byte("48;5;190"), 590 | []byte("48;5;191"), 591 | []byte("48;5;192"), 592 | []byte("48;5;193"), 593 | []byte("48;5;194"), 594 | []byte("48;5;195"), 595 | []byte("48;5;196"), 596 | []byte("48;5;197"), 597 | []byte("48;5;198"), 598 | []byte("48;5;199"), 599 | []byte("48;5;200"), 600 | []byte("48;5;201"), 601 | []byte("48;5;202"), 602 | []byte("48;5;203"), 603 | []byte("48;5;204"), 604 | []byte("48;5;205"), 605 | []byte("48;5;206"), 606 | []byte("48;5;207"), 607 | []byte("48;5;208"), 608 | []byte("48;5;209"), 609 | []byte("48;5;210"), 610 | []byte("48;5;211"), 611 | []byte("48;5;212"), 612 | []byte("48;5;213"), 613 | []byte("48;5;214"), 614 | []byte("48;5;215"), 615 | []byte("48;5;216"), 616 | []byte("48;5;217"), 617 | []byte("48;5;218"), 618 | []byte("48;5;219"), 619 | []byte("48;5;220"), 620 | []byte("48;5;221"), 621 | []byte("48;5;222"), 622 | []byte("48;5;223"), 623 | []byte("48;5;224"), 624 | []byte("48;5;225"), 625 | []byte("48;5;226"), 626 | []byte("48;5;227"), 627 | []byte("48;5;228"), 628 | []byte("48;5;229"), 629 | []byte("48;5;230"), 630 | []byte("48;5;231"), 631 | []byte("48;5;232"), 632 | []byte("48;5;233"), 633 | []byte("48;5;234"), 634 | []byte("48;5;235"), 635 | []byte("48;5;236"), 636 | []byte("48;5;237"), 637 | []byte("48;5;238"), 638 | []byte("48;5;239"), 639 | []byte("48;5;240"), 640 | []byte("48;5;241"), 641 | []byte("48;5;242"), 642 | []byte("48;5;243"), 643 | []byte("48;5;244"), 644 | []byte("48;5;245"), 645 | []byte("48;5;246"), 646 | []byte("48;5;247"), 647 | []byte("48;5;248"), 648 | []byte("48;5;249"), 649 | []byte("48;5;250"), 650 | []byte("48;5;251"), 651 | []byte("48;5;252"), 652 | []byte("48;5;253"), 653 | []byte("48;5;254"), 654 | []byte("48;5;255"), 655 | } 656 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2020 The golang.design Initiative Authors. 2 | // All rights reserved. Use of this source code is governed 3 | // by a GNU GPLv3 license that can be found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "flag" 10 | "fmt" 11 | "io" 12 | "io/ioutil" 13 | "log" 14 | "os" 15 | "os/exec" 16 | "os/signal" 17 | "strings" 18 | "syscall" 19 | "time" 20 | 21 | "golang.design/x/bench/internal/lock" 22 | "golang.design/x/bench/internal/stat" 23 | "golang.design/x/bench/internal/term" 24 | ) 25 | 26 | func usage() { 27 | fmt.Fprintf(os.Stderr, `usage: bench [options] 28 | options for daemon usage: 29 | -daemon 30 | run bench service 31 | -list 32 | print current and pending commands 33 | 34 | options for significant tests: 35 | -delta-test test 36 | significance test to apply to delta: utest, ttest, or none (default "utest") 37 | -alpha α 38 | consider change significant if p < α (default 0.05) 39 | -geomean 40 | print the geometric mean of each file (default false) 41 | -split labels 42 | split benchmarks by labels (default "pkg,goos,goarch") 43 | -sort order 44 | sort by order: [-]delta, [-]name, none (default "none") 45 | 46 | options for running benchmarks: 47 | -v go test 48 | the -v flag from go test, (default false) 49 | -name go test 50 | the -bench flag from go test (default .) (default ".") 51 | -count go test 52 | the -count flag from go test (default 10) (default 10) 53 | -time go test 54 | the -benchtime flag from go test (default unset) 55 | -cpuprocs go test 56 | the -cpu flag to go test (default unset) 57 | 58 | options for performance locking 59 | -shared 60 | acquire lock in shared mode (default exclusive mode) 61 | -cpufreq percent 62 | set CPU frequency to percent between the min and max (default 90) 63 | while running command, or "none" for no adjustment 64 | `) 65 | os.Exit(2) 66 | } 67 | 68 | var ( 69 | flagDaemon *bool 70 | flagList *bool 71 | 72 | flagDeltaTest *string 73 | flagAlpha *float64 74 | flagGeomean *bool 75 | flagSplit *string 76 | flagSort *string 77 | 78 | flagShared *bool 79 | flagCPUFreq *lock.CpufreqFlag 80 | 81 | flagVerbose *bool 82 | flagName *string 83 | flagCount *int 84 | flagTime *string 85 | flagCPUProcs *string 86 | ) 87 | 88 | func main() { 89 | log.SetPrefix("bench: ") 90 | log.SetFlags(0) 91 | flag.Usage = usage 92 | 93 | // daemon args 94 | flagDaemon = flag.Bool("daemon", false, "run bench service") 95 | flagList = flag.Bool("list", false, "print current and pending commands") 96 | 97 | // benchstat args 98 | flagDeltaTest = flag.String("delta-test", "utest", "significance `test` to apply to delta: utest, ttest, or none") 99 | flagAlpha = flag.Float64("alpha", 0.05, "consider change significant if p < `α`") 100 | flagGeomean = flag.Bool("geomean", false, "print the geometric mean of each file") 101 | flagSplit = flag.String("split", "pkg,goos,goarch", "split benchmarks by `labels`") 102 | flagSort = flag.String("sort", "none", "sort by `order`: [-]delta, [-]name, none") 103 | 104 | // perflock flags 105 | flagShared = flag.Bool("shared", false, "acquire lock in shared mode (default exclusive mode)") 106 | flagCPUFreq = &lock.CpufreqFlag{Percent: 90} 107 | flag.Var(flagCPUFreq, "cpufreq", "set CPU frequency to `percent` between the min and max\n\twhile running command, or \"none\" for no adjustment") 108 | 109 | // go test args 110 | flagVerbose = flag.Bool("v", false, "the -v flag from `go test`, (default false)") 111 | flagName = flag.String("name", ".", "the -bench flag from `go test` (default .)") 112 | flagCount = flag.Int("count", 10, "the -count flag from `go test` (default 10)") 113 | flagTime = flag.String("time", "", "the -benchtime flag from `go test` (default unset)") 114 | flagCPUProcs = flag.String("cpuprocs", "", "the -cpu flag to `go test` (default unset)") 115 | flag.Parse() 116 | 117 | if *flagDaemon { 118 | if flag.NArg() > 0 { 119 | flag.Usage() 120 | os.Exit(2) 121 | } 122 | lock.RunDaemon() 123 | return 124 | } 125 | if *flagList { 126 | if flag.NArg() > 0 { 127 | flag.Usage() 128 | os.Exit(2) 129 | } 130 | c := lock.NewClient() 131 | if c == nil { 132 | log.Fatal("Is the bench daemon running?") 133 | } 134 | list := c.List() 135 | if len(list) == 0 { 136 | log.Println("daemon is running but no running benchmarks.") 137 | return 138 | } 139 | for _, l := range list { 140 | log.Println(l) 141 | } 142 | return 143 | } 144 | 145 | if flag.NArg() > 0 { 146 | runCompare() 147 | return 148 | } 149 | 150 | // prepare go test command 151 | args := []string{ 152 | "go", 153 | "test", 154 | "-run=^$", 155 | } 156 | if *flagVerbose { 157 | args = append(args, "-v") 158 | } 159 | args = append(args, fmt.Sprintf("-bench=%s", *flagName)) 160 | if *flagCount <= 0 { 161 | *flagCount = 10 162 | } 163 | args = append(args, fmt.Sprintf("-count=%d", *flagCount)) 164 | if *flagTime != "" { 165 | args = append(args, fmt.Sprintf("-benchtime=%s", *flagTime)) 166 | } 167 | if *flagCPUProcs != "" { 168 | args = append(args, fmt.Sprintf("-cpu=%s", *flagCPUProcs)) 169 | } 170 | 171 | // acquire lock 172 | c := lock.NewClient() 173 | if c == nil { 174 | log.Printf(term.Red("run benchmarks without performance locking...")) 175 | } else { 176 | if !c.Acquire(*flagShared, true, strings.Join(args, " ")) { 177 | list := c.List() 178 | log.Printf("Waiting for lock...\n") 179 | for _, l := range list { 180 | log.Println(l) 181 | } 182 | c.Acquire(*flagShared, false, strings.Join(args, " ")) 183 | } 184 | if !*flagShared && flagCPUFreq.Percent >= 0 { 185 | c.SetCPUFreq(flagCPUFreq.Percent) 186 | log.Print(term.Gray(fmt.Sprintf("run benchmarks under %d%% cpufreq...", flagCPUFreq.Percent))) 187 | } 188 | } 189 | 190 | // Ignore SIGINT and SIGQUIT so they pass through to the 191 | // child. 192 | signal.Notify(make(chan os.Signal), os.Interrupt, syscall.SIGQUIT) 193 | 194 | // run bench 195 | runBench(args) 196 | } 197 | 198 | // shellEscape escapes a single shell token. 199 | func shellEscape(x string) string { 200 | if len(x) == 0 { 201 | return "''" 202 | } 203 | for _, r := range x { 204 | if 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || '0' <= r && r <= '9' || strings.ContainsRune("@%_-+:,./", r) { 205 | continue 206 | } 207 | // Unsafe character. 208 | return "'" + strings.Replace(x, "'", "'\"'\"'", -1) + "'" 209 | } 210 | return x 211 | } 212 | 213 | // shellEscapeList escapes a list of shell tokens. 214 | func shellEscapeList(xs []string) string { 215 | out := make([]string, len(xs)) 216 | for i, x := range xs { 217 | out[i] = shellEscape(x) 218 | } 219 | return strings.Join(out, " ") 220 | } 221 | 222 | var deltaTestNames = map[string]stat.DeltaTest{ 223 | "none": stat.NoDeltaTest, 224 | "u": stat.UTest, 225 | "u-test": stat.UTest, 226 | "utest": stat.UTest, 227 | "t": stat.TTest, 228 | "t-test": stat.TTest, 229 | "ttest": stat.TTest, 230 | } 231 | 232 | func runCompare() { 233 | c := &stat.Collection{ 234 | Alpha: *flagAlpha, 235 | AddGeoMean: *flagGeomean, 236 | DeltaTest: deltaTestNames[strings.ToLower(*flagDeltaTest)], 237 | } 238 | 239 | for _, file := range flag.Args() { 240 | f, err := os.Open(file) 241 | if err != nil { 242 | log.Print(err) 243 | flag.Usage() 244 | os.Exit(2) 245 | } 246 | defer f.Close() 247 | 248 | if err := c.AddFile(file, f); err != nil { 249 | log.Fatal(err) 250 | } 251 | } 252 | 253 | tables := c.Tables() 254 | var buf bytes.Buffer 255 | stat.FormatText(&buf, tables) 256 | os.Stdout.Write(buf.Bytes()) 257 | } 258 | 259 | func runBench(args []string) { 260 | log.Print(strings.Join(args, " ")) 261 | cmd := exec.Command(args[0], args[1:]...) 262 | stdout, err := cmd.StdoutPipe() 263 | if err != nil { 264 | log.Fatal(err) 265 | } 266 | stderr, err := cmd.StderrPipe() 267 | if err != nil { 268 | log.Fatal(err) 269 | } 270 | doneCh := make(chan []byte) 271 | go func() { 272 | data := []byte{} 273 | buf := make([]byte, 1024) 274 | for { 275 | n, err := stdout.Read(buf) 276 | if err != nil { 277 | if err != io.EOF { 278 | fmt.Fprintf(os.Stderr, "%v", err) 279 | } 280 | doneCh <- data 281 | close(doneCh) 282 | return 283 | } 284 | fmt.Fprintf(os.Stdout, string(buf[:n])) 285 | data = append(data, buf[:n]...) 286 | } 287 | }() 288 | errCh := make(chan error) 289 | go func() { 290 | buf := make([]byte, 1024) 291 | for { 292 | n, err := stderr.Read(buf) 293 | if err != nil { 294 | if err == io.EOF { 295 | errCh <- nil 296 | } else { 297 | errCh <- err 298 | } 299 | close(errCh) 300 | return 301 | } 302 | fmt.Fprintf(os.Stderr, string(buf[:n])) 303 | } 304 | }() 305 | 306 | err = cmd.Start() 307 | switch err := err.(type) { 308 | case *exec.ExitError: 309 | status := err.Sys().(syscall.WaitStatus) 310 | if status.Exited() { 311 | os.Exit(status.ExitStatus()) 312 | } 313 | log.Fatal(err) 314 | } 315 | 316 | var results []byte 317 | select { 318 | case err = <-errCh: 319 | if err != nil { // benchmark was interrupted or not success, exit. 320 | os.Exit(2) 321 | return 322 | } 323 | case results = <-doneCh: 324 | } 325 | 326 | // do nothing if no tests were ran. 327 | if strings.Index(string(results), "no Go files") != -1 || 328 | strings.Index(string(results), "no test files") != -1 { 329 | return 330 | } 331 | 332 | // Note that we should avoid using : in filename, because it is not 333 | // supported on Windows file systems. 334 | fname := "bench-" + time.Now().Format("2006-01-02-15-04-05") + ".txt" 335 | err = ioutil.WriteFile(fname, results, 0644) 336 | if err != nil { 337 | // try again, maybe the user was too fast? 338 | err = ioutil.WriteFile(fname, results, 0644) 339 | if err != nil { 340 | log.Fatal("cannot save benchmark result to your disk.") 341 | } 342 | } 343 | log.Printf("results are saved to file: ./%s\n\n", fname) 344 | 345 | computeStat(results) 346 | } 347 | 348 | var sortNames = map[string]stat.Order{ 349 | "none": nil, 350 | "name": stat.ByName, 351 | "delta": stat.ByDelta, 352 | } 353 | 354 | func computeStat(data []byte) { 355 | sortName := *flagSort 356 | reverse := false 357 | if strings.HasPrefix(sortName, "-") { 358 | reverse = true 359 | sortName = sortName[1:] 360 | } 361 | order, _ := sortNames[sortName] 362 | c := &stat.Collection{ 363 | Alpha: *flagAlpha, 364 | AddGeoMean: *flagGeomean, 365 | DeltaTest: deltaTestNames[strings.ToLower(*flagDeltaTest)], 366 | } 367 | 368 | if *flagSplit != "" { 369 | c.SplitBy = strings.Split(*flagSplit, ",") 370 | } 371 | if order != nil { 372 | if reverse { 373 | order = stat.Reverse(order) 374 | } 375 | c.Order = order 376 | } 377 | c.AddData("", data) 378 | tables := c.Tables() 379 | var buf bytes.Buffer 380 | stat.FormatText(&buf, tables) 381 | fmt.Print(buf.String()) 382 | } 383 | --------------------------------------------------------------------------------