├── .coveragerc ├── .gitignore ├── .travis.yml ├── COPYING ├── ChangeLog ├── README.md ├── bin └── exitmap ├── doc ├── HACKING.md ├── logo.png └── logo.svg ├── requirements-dev.txt ├── requirements.txt ├── src ├── command.py ├── error.py ├── eventhandler.py ├── exitmap.py ├── modules │ ├── __init__.py │ ├── checktest.py │ ├── cloudflared.py │ ├── dnspoison.py │ ├── dnssec.py │ ├── patchingCheck.py │ ├── rtt.py │ └── testfds.py ├── relayselector.py ├── selectors34.py ├── six.py ├── stats.py ├── torsocks.py └── util.py └── test ├── run_tests.py ├── test_relayselector.py ├── test_settings.cfg ├── test_stats.py ├── test_torsocks.py └── test_util.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | omit = 3 | src/run_tests.py 4 | src/test_* 5 | src/modules/* 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .coverage 3 | venv/ 4 | __pycache__ 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: 2 | - python 3 | python: 4 | - "2.7" 5 | - "3.4" 6 | - "3.5" 7 | - "pypy" 8 | install: 9 | - pip install -r requirements.txt 10 | - pip install -r requirements-dev.txt 11 | - pip install coveralls 12 | script: 13 | - py.test --cov-report term-missing --cov=src test 14 | after_success: 15 | coveralls 16 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 2020-11-23: Changes in version 2020-11-23: 2 | - Thanks to Kushal Das, the code base is now Python 3. 3 | 4 | 2019-05-30: Changes in version 2019-05-30: 5 | - Resemble HTTP headers of latest Tor Browser. 6 | - Code maintenance. 7 | - Update documentation. 8 | 9 | 2016-04-21: Changes in version 2016.04.21: 10 | - Add command line option '-E' for file containing exit fingerprints. 11 | - Add command line option '-n' for random delays between circuit creations. 12 | - Add username to data directory, allowing multiple users on a single 13 | machine to run exitmap in parallel. 14 | - Make dnspoison module populate its A records automatically. 15 | - Add cloudflared module to check if a site is behind CloudFlare. 16 | - Add rtt module to measure round-trip times from an exit to a destination. 17 | Thanks to Zack Weinberg for the code. 18 | - Add dnssec module to check if an exit relay validates DNSSEC. 19 | - Improved logging now shows module names and Stem's log messages. 20 | - Add command line option '-o' to log to file. 21 | 22 | 2015-08-23: Changes in version 2015.08.23: 23 | - Exclude bad exits by default when selecting exit relays. 24 | - Pass torifying execution environments to modules. 25 | - Set PathsNeededToBuildCircuits to 0.95. 26 | - Replace mysocks/SocksiPy with home-made torsocks. 27 | 28 | 2015-04-06: Changes in version 2015.04.06: 29 | - Publish first release. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![exitmap logo](doc/logo.png) 2 | 3 | **This repository is unmaintained. 4 | Refer to [The Tor Project's fork](https://gitlab.torproject.org/tpo/network-health/exitmap).** 5 | 6 | Overview 7 | -------- 8 | 9 | Exitmap is a fast and modular Python-based scanner for 10 | [Tor](https://www.torproject.org) exit relays. Exitmap modules implement tasks 11 | that are run over (a subset of) all exit relays. If you have a background in 12 | functional programming, think of exitmap as a `map()` interface for Tor exit 13 | relays: Modules can perform any TCP-based networking task like fetching a web 14 | page, uploading a file, connecting to an SSH server, or joining an IRC channel. 15 | 16 | In practice, exitmap is useful to monitor the reliability and trustworthiness of 17 | exit relays. The Tor Project uses exitmap to check for false negatives on the 18 | Tor Project's [check](https://check.torproject.org) service and to find 19 | [malicious exit relays](http://www.cs.kau.se/philwint/spoiled_onions). It is 20 | easy to develop new modules for exitmap; just have a look at the file HACKING in 21 | the doc/ directory or check out one of the existing modules. 22 | 23 | Exitmap uses [Stem](https://stem.torproject.org) to create circuits to all given 24 | exit relays. Each time tor notifies exitmap of an established circuit, a module 25 | is invoked for the newly established circuit. Modules can be pure Python 26 | scripts or executables. For executables, 27 | [torsocks](https://github.com/dgoulet/torsocks/) is necessary. 28 | 29 | Finally, note that exitmap is a network measurement tool and of little use to 30 | ordinary Tor users. The Tor Project is already running the tool regularly. 31 | More exitmap scans just cause unnecessary network load. The only reason exitmap 32 | is publicly available is because its source code and design might be of interest 33 | to some. 34 | 35 | Installation 36 | ------------ 37 | 38 | Exitmap uses the library Stem to communicate with Tor. There are 39 | [plenty of ways](https://stem.torproject.org/download.html) to install Stem. 40 | The easiest might be to use pip in combination with the provided 41 | requirements.txt file: 42 | 43 | $ pip install -r requirements.txt 44 | 45 | Running exitmap 46 | --------------- 47 | 48 | The only argument exitmap requires is the name of a module. For example, you 49 | can run exitmap with the checktest module by running: 50 | 51 | $ ./bin/exitmap checktest 52 | 53 | The command line output will then show you how Tor bootstraps, the output of the 54 | checktest module, and a scan summary. If you don't need three hops and prefer 55 | to use two hops with a static first hop, run: 56 | 57 | $ ./bin/exitmap --first-hop 1234567890ABCDEF1234567890ABCDEF12345678 checktest 58 | 59 | To run the same test over German exit relays only, execute: 60 | 61 | $ ./bin/exitmap --country DE --first-hop 1234567890ABCDEF1234567890ABCDEF12345678 checktest 62 | 63 | If you want to pause for five seconds in between circuit creations to reduce the 64 | load on the Tor network and the scanning destination, run: 65 | 66 | $ ./bin/exitmap --build-delay 5 checktest 67 | 68 | Note that `1234567890ABCDEF1234567890ABCDEF12345678` is a pseudo fingerprint 69 | that you should replace with an exit relay that you control. 70 | 71 | To learn more about all of exitmap's options, run: 72 | 73 | $ ./bin/exitmap --help 74 | 75 | Exitmap comes with batteries included, providing the following modules: 76 | 77 | * testfds: Tests if an exit relay is able to fetch the content of a simple 78 | web page. If an exit relay is unable to do that, it might not have enough 79 | file descriptors available. 80 | * checktest: Attempts to find false negatives in the Tor Project's 81 | [check](https://check.torproject.org) service. 82 | * dnspoison: Attempts to resolve several domains and compares the received DNS A 83 | records to the expected records. 84 | * dnssec: Detects exit relays whose resolver does not validate DNSSEC. 85 | * patchingCheck: Checks for file tampering. 86 | * cloudflared: Checks if a web site returns a CloudFlare CAPTCHA. 87 | * rtt: Measure round-trip times through an exit to various destinations. 88 | 89 | Configuration 90 | ------------- 91 | 92 | By default, exitmap tries to read the file .exitmaprc in your home directory. 93 | The file accepts all command line options, but you have to replace minuses with 94 | underscores. Here is an example: 95 | 96 | [Defaults] 97 | first_hop = 1234567890ABCDEF1234567890ABCDEF12345678 98 | verbosity = debug 99 | build_delay = 1 100 | analysis_dir = /path/to/exitmap_scans 101 | 102 | Alternatives 103 | ------------ 104 | 105 | Don't like exitmap? Then have a look at 106 | [tortunnel](http://www.thoughtcrime.org/software/tortunnel/), 107 | [SoaT](https://gitweb.torproject.org/torflow.git/tree/NetworkScanners/ExitAuthority/README.ExitScanning), 108 | [torscanner](https://code.google.com/p/torscanner/), 109 | [DetecTor](http://detector.io/DetecTor.html), or 110 | [SelekTOR](https://www.dazzleships.net/selektor-for-linux/). 111 | 112 | Tests 113 | ----- 114 | 115 | Before submitting pull requests, please make sure that all unit tests pass by 116 | running: 117 | 118 | $ pip install -r requirements-dev.txt 119 | $ py.test --cov-report term-missing --cov-config .coveragerc --cov=src test 120 | 121 | Feedback 122 | -------- 123 | 124 | Contact: Philipp Winter 125 | OpenPGP fingerprint: `B369 E7A2 18FE CEAD EB96 8C73 CF70 89E3 D7FD C0D0` 126 | -------------------------------------------------------------------------------- /bin/exitmap: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2013, 2014, 2016 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | 20 | import os 21 | import sys 22 | 23 | current_path = os.path.dirname(__file__) 24 | 25 | src_path = os.path.abspath(os.path.join(current_path, "../src")) 26 | 27 | sys.path.insert(0, src_path) 28 | 29 | from exitmap import main 30 | 31 | try: 32 | sys.exit(main()) 33 | except KeyboardInterrupt: 34 | sys.exit(1) 35 | -------------------------------------------------------------------------------- /doc/HACKING.md: -------------------------------------------------------------------------------- 1 | Hacking exitmap modules 2 | ======================= 3 | 4 | So, you are interested in hacking new `exitmap` modules? Modules are Python 5 | files in the directory src/modules/. When invoked, a module engages in some 6 | sort of TCP-based network interaction. Examples are logging into a remote 7 | machine, requesting a web page, or downloading a file over FTP. The module is 8 | then run over (a subset of) all Tor exit relays whose exit policy works for the 9 | module. To get an idea of what a module looks like in practice, have a look at 10 | the existing files in src/modules/. 11 | 12 | From an implementation point of view, there are two types of modules: 13 | 14 | 1. There are modules that are implemented in pure Python. They don't use any 15 | external programs such as `wget`. As a result, you are limited to whatever 16 | Python offers, e.g., `urllib2` for web interaction. The network traffic 17 | generated by these modules is transparently tunneled over SOCKS using the 18 | code in torsocks.py. 19 | 20 | 2. Modules can invoke external tools such `gnutls-cli`, e.g., to fetch X.509 21 | certificates. Similarly, a patched version of `torsocks` is used to 22 | transparently tunnel this type of network traffic over Tor's SOCKS port. 23 | 24 | The function signature of modules is: 25 | 26 | def probe(exit_desc, run_python_over_tor, run_cmd_over_tor) 27 | 28 | The arguments are: 29 | 30 | 1. `exit_desc`: An object of type 31 | `stem.descriptor.server_descriptor.RelayDescriptor`. 32 | 33 | 2. `run_python_over_tor`: Expects a function (as first argument) and its 34 | arguments (as subsequent arguments). The function's network interaction is 35 | then routed over Tor. 36 | 37 | 3. `run_cmd_over_tor`: Expects a command (as first argument) and its parameters 38 | (as subsequent arguments). The command's network interaction is then routed 39 | over Tor using `torsocks`. 40 | 41 | Finally, you must define the global variable `destinations` in your module. It 42 | determines the destinations---as tuples---your module will connect to. 43 | `Exitmap` must know this to select exit relays whose exit policy matches your 44 | module. Here's an example: 45 | 46 | destinations = [("www.example.com", 80), ("smtp.example.com", 25)] 47 | -------------------------------------------------------------------------------- /doc/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NullHypothesis/exitmap/81550298f80496a5122b9335f02ae2fad23d3bc8/doc/logo.png -------------------------------------------------------------------------------- /doc/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 14 | 16 | 22 | 27 | 28 | 34 | 39 | 40 | 42 | 50 | 51 | 53 | 61 | 62 | 63 | 65 | 66 | 68 | image/svg+xml 69 | 71 | 72 | 73 | 74 | 75 | 78 | exitmap 93 | 98 | 102 | 107 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | pytest>=3.6 2 | pytest-cov 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | dnspython 2 | stem 3 | pysocks 4 | -------------------------------------------------------------------------------- /src/command.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2016 Philipp Winter 2 | # 3 | # This file is part of exitmap. 4 | # 5 | # exitmap is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # exitmap is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with exitmap. If not, see . 17 | 18 | """ 19 | Provides and API to execute system commands over torsocks. 20 | """ 21 | 22 | import os 23 | import socket 24 | import threading 25 | import subprocess 26 | import tempfile 27 | import pprint 28 | import logging 29 | 30 | import util 31 | import torsocks 32 | import error 33 | 34 | log = logging.getLogger(__name__) 35 | 36 | 37 | def run_python_over_tor(queue, circ_id, socks_port): 38 | """ 39 | Returns a closure to route a Python function's network traffic over Tor. 40 | """ 41 | 42 | def closure(func, *args): 43 | """ 44 | Route the given Python function's network traffic over Tor. 45 | We temporarily monkey-patch socket.socket using our torsocks 46 | module, and reset it once the function returns. 47 | """ 48 | try: 49 | with torsocks.MonkeyPatchedSocket(queue, circ_id, socks_port): 50 | func(*args) 51 | except (error.SOCKSv5Error, socket.error) as err: 52 | log.info(err) 53 | return 54 | 55 | return closure 56 | 57 | 58 | class Command(object): 59 | 60 | """ 61 | Provide an abstraction for a shell command which is to be run. 62 | """ 63 | 64 | def __init__(self, queue, circ_id, socks_port): 65 | 66 | self.process = None 67 | self.stdout = None 68 | self.stderr = None 69 | self.output_callback = None 70 | self.queue = queue 71 | self.circ_id = circ_id 72 | self.socks_port = socks_port 73 | 74 | def invoke_process(self, command): 75 | """ 76 | Run the command and wait for it to finish. 77 | 78 | If a callback was specified, it is called with the process' output as 79 | argument and with a function which can be used to kill the process. 80 | """ 81 | 82 | # Start process and redirect its stderr to stdout. That makes it more 83 | # convenient for us to parse the output. 84 | 85 | self.process = subprocess.Popen( 86 | command, 87 | env=os.environ, 88 | stdout=subprocess.PIPE, 89 | stderr=subprocess.STDOUT, 90 | ) 91 | 92 | if self.output_callback: 93 | 94 | # Read the process' output line by line and pass it to the module's 95 | # callback function. 96 | 97 | keep_reading = True 98 | while keep_reading: 99 | 100 | line = self.process.stdout.readline() 101 | if not line: 102 | break 103 | else: 104 | line = line.strip() 105 | 106 | # Look for torsocks' source port before we pass the line on 107 | # to the module. 108 | 109 | pattern = "Connection on fd [0-9]+ originating " \ 110 | "from [^:]+:([0-9]{1,5})" 111 | port = util.extract_pattern(line, pattern) 112 | 113 | if port: 114 | self.queue.put([self.circ_id, ("127.0.0.1", int(port))]) 115 | 116 | keep_reading = self.output_callback(line, self.process.kill) 117 | 118 | # Wait for the process to finish. 119 | 120 | self.stdout, self.stderr = self.process.communicate() 121 | 122 | def execute(self, command, timeout=10, output_callback=None): 123 | """ 124 | Run a shell command in a dedicated process. 125 | """ 126 | 127 | command = ["torsocks"] + command 128 | self.output_callback = output_callback 129 | 130 | # We run the given command in a separate thread. The main thread will 131 | # kill the process if it does not finish before the given timeout. 132 | 133 | with tempfile.NamedTemporaryFile(prefix="torsocks_") as fd: 134 | 135 | log.debug("Created temporary torsocks config file %s" % fd.name) 136 | os.environ["TORSOCKS_CONF_FILE"] = fd.name 137 | os.environ["TORSOCKS_LOG_LEVEL"] = "5" 138 | 139 | fd.write("TorPort %d\n" % self.socks_port) 140 | fd.write("TorAddress 127.0.0.1\n") 141 | fd.flush() 142 | 143 | log.debug("Invoking \"%s\" in environment:\n%s" % 144 | (" ".join(command), pprint.pformat(dict(os.environ)))) 145 | 146 | thread = threading.Thread(target=self.invoke_process, 147 | args=(command,)) 148 | thread.daemon = True 149 | thread.start() 150 | thread.join(timeout) 151 | 152 | # Attempt to kill the process if it did not finish in time. 153 | 154 | if thread.is_alive(): 155 | log.debug("Killing process after %d seconds." % timeout) 156 | self.process.kill() 157 | thread.join() 158 | 159 | return self.stdout, self.stderr 160 | 161 | 162 | # Alias class name to provide more intuitive interface. 163 | new = Command 164 | -------------------------------------------------------------------------------- /src/error.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2015 Philipp Winter 2 | # 3 | # This file is part of exitmap. 4 | # 5 | # exitmap is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # exitmap is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with exitmap. If not, see . 17 | 18 | """ 19 | Exports custom exceptions. 20 | """ 21 | 22 | 23 | class ExitSelectionError(Exception): 24 | 25 | """ 26 | Represents an error during selection of exit relays. 27 | """ 28 | 29 | pass 30 | 31 | 32 | class PathSelectionError(Exception): 33 | 34 | """ 35 | Represents an error during selection of a path for a circuit. 36 | """ 37 | 38 | pass 39 | 40 | 41 | class SOCKSv5Error(Exception): 42 | 43 | """ 44 | Represents an error while negotiating SOCKSv5. 45 | """ 46 | 47 | pass 48 | -------------------------------------------------------------------------------- /src/eventhandler.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2016 Philipp Winter 2 | # 3 | # This file is part of exitmap. 4 | # 5 | # exitmap is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # exitmap is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with exitmap. If not, see . 17 | 18 | """ 19 | Handles Tor controller events. 20 | """ 21 | 22 | import sys 23 | import functools 24 | import threading 25 | import multiprocessing 26 | import logging 27 | 28 | import stem 29 | from stem import StreamStatus 30 | from stem import CircStatus 31 | 32 | import command 33 | import util 34 | 35 | log = logging.getLogger(__name__) 36 | 37 | 38 | def get_relay_desc(controller, fpr): 39 | """ 40 | Return the descriptor for the given relay fingerprint." 41 | """ 42 | 43 | desc = None 44 | try: 45 | desc = controller.get_server_descriptor(relay=fpr) 46 | except stem.DescriptorUnavailable as err: 47 | log.warning("Descriptor for %s not available: %s" % (fpr, err)) 48 | except stem.ControllerError as err: 49 | log.warning("Unable to query for %d: %s" % (fpr, err)) 50 | except ValueError: 51 | log.warning("%s is malformed. Is it a relay fingerprint?" % fpr) 52 | 53 | return desc 54 | 55 | 56 | class Attacher(object): 57 | 58 | """ 59 | Attaches streams to circuits. 60 | """ 61 | 62 | def __init__(self, controller): 63 | 64 | # Maps port to function that attached a stream to a circuit. 65 | 66 | self.unattached = {} 67 | self.controller = controller 68 | 69 | def prepare(self, port, circuit_id=None, stream_id=None): 70 | """ 71 | Prepare for attaching a stream to a circuit. 72 | 73 | If we already have the corresponding stream/circuit, we can attach it 74 | now. Otherwise, the method _attach() is partially executed and stored, 75 | so it can be attached later. 76 | """ 77 | 78 | assert ((circuit_id is not None) and (stream_id is None)) or \ 79 | ((circuit_id is None) and (stream_id is not None)) 80 | 81 | # Check if we can already attach. 82 | 83 | if port in self.unattached: 84 | attach = self.unattached[port] 85 | 86 | if circuit_id: 87 | attach(circuit_id=circuit_id) 88 | else: 89 | attach(stream_id=stream_id) 90 | 91 | del self.unattached[port] 92 | else: 93 | # We maintain a dictionary of source ports that point to their 94 | # respective attaching function. At this point we only know either 95 | # the stream or the circuit ID, so we store a partially executed 96 | # function. 97 | 98 | if circuit_id: 99 | partially_attached = functools.partial(self._attach, 100 | circuit_id=circuit_id) 101 | self.unattached[port] = partially_attached 102 | else: 103 | partially_attached = functools.partial(self._attach, 104 | stream_id=stream_id) 105 | self.unattached[port] = partially_attached 106 | 107 | log.debug("Pending attachers: %d." % len(self.unattached)) 108 | 109 | def _attach(self, stream_id=None, circuit_id=None): 110 | """ 111 | Attach a stream to a circuit. 112 | """ 113 | 114 | log.debug("Attempting to attach stream %s to circuit %s." % 115 | (stream_id, circuit_id)) 116 | 117 | try: 118 | self.controller.attach_stream(stream_id, circuit_id) 119 | except stem.OperationFailed as err: 120 | log.warning("Failed to attach stream because: %s" % err) 121 | 122 | 123 | def module_closure(queue, module, circ_id, *module_args, **module_kwargs): 124 | """ 125 | Return function that runs the module and then informs event handler. 126 | """ 127 | 128 | def func(): 129 | """ 130 | Run the module and then inform the event handler. 131 | 132 | The invoking process keeps track of which circuits finished. Once we 133 | are done, we send a signal over the queue to let the process know. 134 | """ 135 | 136 | try: 137 | module(*module_args, **module_kwargs) 138 | 139 | log.debug("Informing event handler that module finished.") 140 | queue.put((circ_id, None)) 141 | except KeyboardInterrupt: 142 | pass 143 | 144 | return func 145 | 146 | 147 | class EventHandler(object): 148 | 149 | """ 150 | Handles asynchronous Tor events. 151 | 152 | The handler processes only stream and circuit events. New streams are 153 | attached to their corresponding circuits since exitmap's Tor process leaves 154 | new streams unattached. 155 | """ 156 | 157 | def __init__(self, controller, module, socks_port, stats, exit_destinations): 158 | 159 | self.stats = stats 160 | self.controller = controller 161 | self.attacher = Attacher(controller) 162 | self.module = module 163 | self.manager = multiprocessing.Manager() 164 | self.queue = self.manager.Queue() 165 | self.socks_port = socks_port 166 | self.exit_destinations = exit_destinations 167 | self.check_finished_lock = threading.Lock() 168 | self.already_finished = False 169 | 170 | queue_thread = threading.Thread(target=self.queue_reader) 171 | queue_thread.daemon = False 172 | queue_thread.start() 173 | 174 | def queue_reader(self): 175 | """ 176 | Read (circuit ID, sockname) tuples from invoked probing modules. 177 | 178 | These tuples are then used to attach streams to their corresponding 179 | circuits. 180 | """ 181 | 182 | log.debug("Starting thread to read from IPC queue.") 183 | 184 | while True: 185 | try: 186 | circ_id, sockname = self.queue.get() 187 | except EOFError: 188 | log.debug("IPC queue terminated.") 189 | break 190 | 191 | # Over the queue, a module can either signal that it finished 192 | # execution (by sending (circ_id,None)) or that it is ready to have 193 | # its stream attached to a circuit (by sending (circ_id,sockname)). 194 | 195 | if sockname is None: 196 | log.debug("Closing finished circuit %s." % circ_id) 197 | try: 198 | self.controller.close_circuit(circ_id) 199 | except stem.InvalidArguments as err: 200 | log.debug("Could not close circuit because: %s" % err) 201 | 202 | self.stats.finished_streams += 1 203 | self.stats.print_progress() 204 | self.check_finished() 205 | else: 206 | log.debug("Read from queue: %s, %s" % (circ_id, str(sockname))) 207 | port = int(sockname[1]) 208 | self.attacher.prepare(port, circuit_id=circ_id) 209 | self.check_finished() 210 | 211 | def check_finished(self): 212 | """ 213 | Check if the scan is finished and if it is, shut down exitmap. 214 | """ 215 | 216 | # This is called from both the queue reader thread and the 217 | # main thread, but (if it detects completion) does things that 218 | # must only happen once. 219 | with self.check_finished_lock: 220 | if self.already_finished: 221 | sys.exit(0) 222 | 223 | # Did all circuits either build or fail? 224 | circs_done = ((self.stats.failed_circuits + 225 | self.stats.successful_circuits) == 226 | self.stats.total_circuits) 227 | 228 | # Was every built circuit attached to a stream? 229 | streams_done = (self.stats.finished_streams >= 230 | (self.stats.successful_circuits - 231 | self.stats.failed_circuits)) 232 | 233 | log.debug("failedCircs=%d, builtCircs=%d, totalCircs=%d, " 234 | "finishedStreams=%d" % (self.stats.failed_circuits, 235 | self.stats.successful_circuits, 236 | self.stats.total_circuits, 237 | self.stats.finished_streams)) 238 | 239 | if circs_done and streams_done: 240 | self.already_finished = True 241 | 242 | for proc in multiprocessing.active_children(): 243 | log.debug("Terminating remaining PID %d." % proc.pid) 244 | proc.terminate() 245 | 246 | if hasattr(self.module, "teardown"): 247 | log.debug("Calling module's teardown() function.") 248 | self.module.teardown() 249 | 250 | log.info(self.stats) 251 | sys.exit(0) 252 | 253 | def new_circuit(self, circ_event): 254 | """ 255 | Invoke a new probing module when a new circuit becomes ready. 256 | """ 257 | 258 | self.stats.update_circs(circ_event) 259 | self.check_finished() 260 | 261 | if circ_event.status not in [CircStatus.BUILT]: 262 | return 263 | 264 | last_hop = circ_event.path[-1] 265 | exit_fpr = last_hop[0] 266 | log.debug("Circuit for exit relay \"%s\" is built. " 267 | "Now invoking probing module." % exit_fpr) 268 | 269 | run_cmd_over_tor = command.Command(self.queue, 270 | circ_event.id, 271 | self.socks_port) 272 | 273 | exit_desc = get_relay_desc(self.controller, exit_fpr) 274 | if exit_desc is None: 275 | self.controller.close_circuit(circ_event.id) 276 | return 277 | 278 | module = module_closure(self.queue, self.module.probe, 279 | circ_event.id, exit_desc, 280 | command.run_python_over_tor(self.queue, 281 | circ_event.id, 282 | self.socks_port), 283 | run_cmd_over_tor, 284 | destinations=self.exit_destinations[exit_fpr]) 285 | 286 | proc = multiprocessing.Process(target=module) 287 | proc.daemon = True 288 | proc.start() 289 | 290 | def new_stream(self, stream_event): 291 | """ 292 | Create a function which is later used to attach a stream to a circuit. 293 | 294 | The attaching cannot be done right now as we do not know the stream's 295 | desired circuit ID at this point. So we set up all we can at this 296 | point and wait for the attaching to be done in queue_reader(). 297 | """ 298 | 299 | if stream_event.status not in [StreamStatus.NEW, 300 | StreamStatus.NEWRESOLVE]: 301 | return 302 | 303 | port = util.get_source_port(str(stream_event)) 304 | if not port: 305 | log.warning("Couldn't extract source port from stream " 306 | "event: %s" % str(stream_event)) 307 | return 308 | 309 | log.debug("Adding attacher for new stream %s." % stream_event.id) 310 | self.attacher.prepare(port, stream_id=stream_event.id) 311 | self.check_finished() 312 | 313 | def new_event(self, event): 314 | """ 315 | Dispatches new Tor controller events to the appropriate handlers. 316 | """ 317 | 318 | if isinstance(event, stem.response.events.CircuitEvent): 319 | self.new_circuit(event) 320 | 321 | elif isinstance(event, stem.response.events.StreamEvent): 322 | self.new_stream(event) 323 | 324 | else: 325 | log.warning("Received unexpected event %s." % str(event)) 326 | -------------------------------------------------------------------------------- /src/exitmap.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2016 Philipp Winter 2 | # 3 | # This file is part of exitmap. 4 | # 5 | # exitmap is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # exitmap is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with exitmap. If not, see . 17 | 18 | """ 19 | Performs a task over (a subset of) all Tor exit relays. 20 | """ 21 | 22 | import sys 23 | import os 24 | import time 25 | import socket 26 | import pkgutil 27 | import argparse 28 | import datetime 29 | import random 30 | import logging 31 | from configparser import ConfigParser 32 | import functools 33 | import pwd 34 | 35 | import stem 36 | import stem.connection 37 | import stem.process 38 | import stem.descriptor 39 | from stem.control import Controller, EventType 40 | 41 | import modules 42 | import error 43 | import util 44 | import relayselector 45 | 46 | from eventhandler import EventHandler 47 | from stats import Statistics 48 | 49 | log = logging.getLogger(__name__) 50 | 51 | 52 | def bootstrap_tor(args): 53 | """ 54 | Invoke a Tor process which is subsequently used by exitmap. 55 | """ 56 | 57 | log.info("Attempting to invoke Tor process in directory \"%s\". This " 58 | "might take a while." % args.tor_dir) 59 | 60 | if not args.first_hop: 61 | log.info("No first hop given. Using randomly determined first " 62 | "hops for circuits.") 63 | 64 | ports = {} 65 | partial_parse_log_lines = functools.partial(util.parse_log_lines, ports) 66 | 67 | try: 68 | proc = stem.process.launch_tor_with_config( 69 | config={ 70 | "SOCKSPort": "auto", 71 | "ControlPort": "auto", 72 | "DataDirectory": args.tor_dir, 73 | "CookieAuthentication": "1", 74 | "LearnCircuitBuildTimeout": "0", 75 | "CircuitBuildTimeout": "40", 76 | "__DisablePredictedCircuits": "1", 77 | "__LeaveStreamsUnattached": "1", 78 | "FetchHidServDescriptors": "0", 79 | "UseMicroDescriptors": "0", 80 | "PathsNeededToBuildCircuits": "0.95", 81 | }, 82 | timeout=300, 83 | take_ownership=True, 84 | completion_percent=75, 85 | init_msg_handler=partial_parse_log_lines, 86 | ) 87 | log.info("Successfully started Tor process (PID=%d)." % proc.pid) 88 | except OSError as err: 89 | log.error("Couldn't launch Tor: %s. Maybe try again?" % err) 90 | sys.exit(1) 91 | 92 | return ports["socks"], ports["control"] 93 | 94 | 95 | def parse_cmd_args(): 96 | """ 97 | Parse and return command line arguments. 98 | """ 99 | 100 | desc = "Perform a task over (a subset of) all Tor exit relays." 101 | parser = argparse.ArgumentParser(description=desc, add_help=False) 102 | 103 | parser.add_argument("-f", "--config-file", type=str, default=None, 104 | help="Path to the configuration file.") 105 | 106 | args, remaining_argv = parser.parse_known_args() 107 | 108 | # First, try to load the configuration file and load its content as our 109 | # defaults. 110 | 111 | if args.config_file: 112 | config_file = args.config_file 113 | else: 114 | home_dir = os.path.expanduser("~") 115 | config_file = os.path.join(home_dir, ".exitmaprc") 116 | 117 | config_parser = ConfigParser() 118 | file_parsed = config_parser.read([config_file]) 119 | if file_parsed: 120 | try: 121 | defaults = dict(config_parser.items("Defaults")) 122 | except ConfigParser.NoSectionError as err: 123 | log.warning("Could not parse config file \"%s\": %s" % 124 | (config_file, err)) 125 | defaults = {} 126 | else: 127 | defaults = {} 128 | 129 | parser = argparse.ArgumentParser(parents=[parser]) 130 | parser.set_defaults(**defaults) 131 | 132 | # Now, load the arguments given over the command line. 133 | 134 | group = parser.add_mutually_exclusive_group() 135 | 136 | group.add_argument("-C", "--country", type=str, default=None, 137 | help="Only probe exit relays of the country which is " 138 | "determined by the given 2-letter country code.") 139 | 140 | group.add_argument("-e", "--exit", type=str, default=None, 141 | help="Only probe the exit relay which has the given " 142 | "20-byte fingerprint.") 143 | 144 | group.add_argument("-E", "--exit-file", type=str, default=None, 145 | help="File containing the 20-byte fingerprints " 146 | "of exit relays to probe, one per line.") 147 | 148 | parser.add_argument("-d", "--build-delay", type=float, default=3, 149 | help="Wait for the given delay (in seconds) between " 150 | "circuit builds. The default is 3.") 151 | 152 | parser.add_argument("-n", "--delay-noise", type=float, default=0, 153 | help="Sample random value in [0, DELAY_NOISE) and " 154 | "randomly add it to or subtract it from the build" 155 | " delay. This randomises the build delay. The " 156 | "default is 0.") 157 | 158 | # Create /tmp/exitmap_tor_datadir-$USER to allow many users to run 159 | # exitmap in parallel. 160 | 161 | tor_directory = "/tmp/exitmap_tor_datadir-" + pwd.getpwuid(os.getuid())[0] 162 | 163 | parser.add_argument("-t", "--tor-dir", type=str, 164 | default=tor_directory, 165 | help="Tor's data directory. If set, the network " 166 | "consensus can be re-used in between scans which " 167 | "speeds up bootstrapping. The default is %s." % 168 | tor_directory) 169 | 170 | parser.add_argument("-a", "--analysis-dir", type=str, 171 | default=None, 172 | help="The directory where analysis results are " 173 | "written to. If the directory is used depends " 174 | "on the module. The default is /tmp.") 175 | 176 | parser.add_argument("-v", "--verbosity", type=str, default="info", 177 | help="Minimum verbosity level for logging. Available " 178 | "in ascending order: debug, info, warning, " 179 | "error, critical). The default is info.") 180 | 181 | parser.add_argument("-i", "--first-hop", type=str, default=None, 182 | help="The 20-byte fingerprint of the Tor relay which " 183 | "is used as first hop. This relay should be " 184 | "under your control.") 185 | 186 | parser.add_argument("-o", "--logfile", type=str, default=None, 187 | help="Filename to which log output should be written " 188 | "to.") 189 | 190 | exits = parser.add_mutually_exclusive_group() 191 | 192 | exits.add_argument("-b", "--bad-exits", action="store_true", 193 | help="Only scan exit relays that have the BadExit " 194 | "flag. By default, only good exits are scanned.") 195 | 196 | exits.add_argument("-l", "--all-exits", action="store_true", 197 | help="Scan all exits, including those that have the " 198 | "BadExit flag. By default, only good exits are " 199 | "scanned.") 200 | 201 | parser.add_argument("-V", "--version", action="version", 202 | version="%(prog)s 2020.11.23") 203 | 204 | parser.add_argument("module", nargs='+', 205 | help="Run the given module (available: %s)." % 206 | ", ".join(get_modules())) 207 | 208 | parser.set_defaults(**defaults) 209 | 210 | return parser.parse_args(remaining_argv) 211 | 212 | 213 | def get_modules(): 214 | """ 215 | Return all modules located in "modules/". 216 | """ 217 | 218 | modules_path = os.path.dirname(modules.__file__) 219 | 220 | return [name for _, name, _ in pkgutil.iter_modules([modules_path])] 221 | 222 | 223 | def main(): 224 | """ 225 | The scanner's entry point. 226 | """ 227 | 228 | stats = Statistics() 229 | args = parse_cmd_args() 230 | 231 | # Create and set the given directories. 232 | 233 | if args.tor_dir and not os.path.exists(args.tor_dir): 234 | os.makedirs(args.tor_dir) 235 | 236 | logging.getLogger("stem").setLevel(logging.__dict__[args.verbosity.upper()]) 237 | log_format = "%(asctime)s %(name)s [%(levelname)s] %(message)s" 238 | logging.basicConfig(format=log_format, 239 | level=logging.__dict__[args.verbosity.upper()], 240 | filename=args.logfile) 241 | 242 | log.debug("Command line arguments: %s" % str(args)) 243 | 244 | socks_port, control_port = bootstrap_tor(args) 245 | controller = Controller.from_port(port=control_port) 246 | stem.connection.authenticate(controller) 247 | 248 | # Redirect Tor's logging to work around the following problem: 249 | # https://bugs.torproject.org/9862 250 | 251 | log.debug("Redirecting Tor's logging to /dev/null.") 252 | controller.set_conf("Log", "err file /dev/null") 253 | 254 | # We already have the current consensus, so we don't need additional 255 | # descriptors or the streams fetching them. 256 | 257 | controller.set_conf("FetchServerDescriptors", "0") 258 | 259 | cached_consensus_path = os.path.join(args.tor_dir, "cached-consensus") 260 | if args.first_hop and (not util.relay_in_consensus(args.first_hop, 261 | cached_consensus_path)): 262 | log.critical("Given first hop \"%s\" not found in consensus. Is it" 263 | " offline?" % args.first_hop) 264 | return 1 265 | 266 | for module_name in args.module: 267 | 268 | if args.analysis_dir is not None: 269 | datestr = time.strftime("%Y-%m-%d_%H:%M:%S%z") + "_" + module_name 270 | util.analysis_dir = os.path.join(args.analysis_dir, datestr) 271 | 272 | try: 273 | run_module(module_name, args, controller, socks_port, stats) 274 | except error.ExitSelectionError as err: 275 | log.error("Failed to run because : %s" % err) 276 | return 0 277 | 278 | 279 | def lookup_destinations(module): 280 | """ 281 | Determine the set of destinations that the module might like to scan. 282 | This removes redundancies and reduces all hostnames to IP addresses. 283 | """ 284 | destinations = set() 285 | addrs = {} 286 | if hasattr(module, 'destinations'): 287 | raw_destinations = module.destinations 288 | if raw_destinations is not None: 289 | for (host, port) in raw_destinations: 290 | if host not in addrs: 291 | addrs[host] = socket.gethostbyname(host) 292 | destinations.add((addrs[host], port)) 293 | 294 | return destinations 295 | 296 | 297 | def select_exits(args, module): 298 | """ 299 | Select exit relays which allow exiting to the module's scan destinations. 300 | 301 | We select exit relays based on their published exit policy. In particular, 302 | we check if the exit relay's exit policy specifies that we can connect to 303 | our intended destination(s). 304 | """ 305 | 306 | before = datetime.datetime.now() 307 | destinations = lookup_destinations(module) 308 | 309 | if args.exit: 310 | # '-e' was used to specify a single exit relay. 311 | requested_exits = [args.exit] 312 | elif args.exit_file: 313 | # '-E' was used to specify a file containing exit relays. 314 | try: 315 | requested_exits = [line.strip() for line in open(args.exit_file)] 316 | except OSError as err: 317 | log.error("Could not read %s: %s", args.exit_file, err.strerror) 318 | sys.exit(1) 319 | except Exception as err: 320 | log.error("Could not read %s: %s", args.exit_file, err) 321 | sys.exit(1) 322 | else: 323 | requested_exits = None 324 | 325 | exit_destinations = relayselector.get_exits( 326 | args.tor_dir, 327 | good_exit = args.all_exits or (not args.bad_exits), 328 | bad_exit = args.all_exits or args.bad_exits, 329 | country_code = args.country, 330 | requested_exits = requested_exits, 331 | destinations = destinations) 332 | 333 | log.debug("Successfully selected exit relays after %s." % 334 | str(datetime.datetime.now() - before)) 335 | 336 | return exit_destinations 337 | 338 | 339 | def run_module(module_name, args, controller, socks_port, stats): 340 | """ 341 | Run an exitmap module over all available exit relays. 342 | """ 343 | 344 | log.info("Running module '%s'." % module_name) 345 | stats.modules_run += 1 346 | 347 | try: 348 | module = __import__("modules.%s" % module_name, fromlist=[module_name]) 349 | except ImportError as err: 350 | log.error("Failed to load module because: %s" % err) 351 | return 352 | 353 | # Let module perform one-off setup tasks. 354 | 355 | if hasattr(module, "setup"): 356 | log.debug("Calling module's setup() function.") 357 | module.setup() 358 | 359 | exit_destinations = select_exits(args, module) 360 | 361 | exit_relays = list(exit_destinations.keys()) 362 | random.shuffle(exit_relays) 363 | 364 | count = len(exit_relays) 365 | stats.total_circuits += count 366 | 367 | if count < 1: 368 | raise error.ExitSelectionError("Exit selection yielded %d exits " 369 | "but need at least one." % count) 370 | 371 | handler = EventHandler(controller, module, socks_port, stats, 372 | exit_destinations=exit_destinations) 373 | 374 | controller.add_event_listener(handler.new_event, 375 | EventType.CIRC, EventType.STREAM) 376 | 377 | duration = count * args.build_delay 378 | log.info("Scan is estimated to take around %s." % 379 | datetime.timedelta(seconds=duration)) 380 | 381 | log.info("Beginning to trigger %d circuit creation(s)." % count) 382 | 383 | iter_exit_relays(exit_relays, controller, stats, args) 384 | 385 | 386 | def sleep(delay, delay_noise): 387 | """ 388 | Sleep in between circuit creations. 389 | 390 | This has two purposes. First, it spreads the load on both the Tor network 391 | and our scanning destination over time. Second, by using random values to 392 | obscure our circuit creation patterns, we hopefully make it harder for a 393 | vigilant adversary to detect our scanning. 394 | """ 395 | 396 | noise = 0 397 | if delay_noise != 0: 398 | noise = random.random() * delay_noise 399 | if random.randint(0, 1): 400 | noise = -noise 401 | 402 | delay += noise 403 | if delay < 0: 404 | delay = 0 405 | 406 | log.debug("Sleeping for %.1fs, then building next circuit." % delay) 407 | time.sleep(delay) 408 | 409 | 410 | def iter_exit_relays(exit_relays, controller, stats, args): 411 | """ 412 | Invoke circuits for all selected exit relays. 413 | """ 414 | 415 | before = datetime.datetime.now() 416 | cached_consensus_path = os.path.join(args.tor_dir, "cached-consensus") 417 | fingerprints = relayselector.get_fingerprints(cached_consensus_path) 418 | count = len(exit_relays) 419 | 420 | # Start building a circuit for every exit relay we got. 421 | 422 | for i, exit_relay in enumerate(exit_relays): 423 | 424 | # Determine the hops in our next circuit. 425 | 426 | if args.first_hop: 427 | hops = [args.first_hop, exit_relay] 428 | else: 429 | all_hops = list(fingerprints) 430 | 431 | try: 432 | all_hops.remove(exit_relay) 433 | except ValueError: 434 | # Catch exception when exit is not in the cached_consensus 435 | pass 436 | first_hop = random.choice(all_hops) 437 | log.debug("Using random first hop %s for circuit." % first_hop) 438 | hops = [first_hop, exit_relay] 439 | 440 | assert len(hops) > 1 441 | 442 | try: 443 | controller.new_circuit(hops) 444 | except stem.ControllerError as err: 445 | stats.failed_circuits += 1 446 | log.debug("Circuit with exit relay \"%s\" could not be " 447 | "created: %s" % (exit_relay, err)) 448 | 449 | if i != (count - 1): 450 | sleep(args.build_delay, args.delay_noise) 451 | 452 | log.info("Done triggering circuit creations after %s." % 453 | str(datetime.datetime.now() - before)) 454 | -------------------------------------------------------------------------------- /src/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NullHypothesis/exitmap/81550298f80496a5122b9335f02ae2fad23d3bc8/src/modules/__init__.py -------------------------------------------------------------------------------- /src/modules/checktest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2013-2017 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | 20 | """ 21 | Module to detect false negatives for . 22 | """ 23 | 24 | import sys 25 | import json 26 | import logging 27 | import urllib.request 28 | import socks 29 | import socket 30 | 31 | from util import exiturl 32 | 33 | import stem.descriptor.server_descriptor as descriptor 34 | 35 | log = logging.getLogger(__name__) 36 | 37 | # exitmap needs this variable to figure out which relays can exit to the given 38 | # destination(s). 39 | 40 | destinations = [("check.torproject.org", 443)] 41 | 42 | 43 | def fetch_page(exit_desc): 44 | """ 45 | Fetch check.torproject.org and see if we are using Tor. 46 | """ 47 | 48 | data = None 49 | url = exiturl(exit_desc.fingerprint) 50 | 51 | try: 52 | data = urllib.request.urlopen("https://check.torproject.org/api/ip", 53 | timeout=10).read() 54 | except Exception as err: 55 | log.debug("urllib.request.urlopen says: %s" % err) 56 | return 57 | 58 | if not data: 59 | return 60 | 61 | try: 62 | check_answer = json.loads(data) 63 | except ValueError as err: 64 | log.warning("Couldn't parse JSON over relay %s: %s" % (url, data)) 65 | return 66 | 67 | check_addr = check_answer["IP"].strip() 68 | if not check_answer["IsTor"]: 69 | log.error("Check thinks %s isn't Tor. Desc addr is %s and check " 70 | "addr is %s." % (url, exit_desc.address, check_addr)) 71 | else: 72 | log.debug("Exit relay %s passed the check test." % url) 73 | 74 | 75 | def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs): 76 | """ 77 | Probe the given exit relay and look for check.tp.o false negatives. 78 | """ 79 | 80 | run_python_over_tor(fetch_page, exit_desc) 81 | 82 | 83 | def main(): 84 | """ 85 | Entry point when invoked over the command line. 86 | """ 87 | 88 | desc = descriptor.ServerDescriptor("") 89 | desc.fingerprint = "bogus" 90 | desc.address = "0.0.0.0" 91 | fetch_page(desc) 92 | 93 | return 0 94 | 95 | 96 | if __name__ == "__main__": 97 | sys.exit(main()) 98 | -------------------------------------------------------------------------------- /src/modules/cloudflared.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2016 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | 20 | """ 21 | Check if a web site returns a CloudFlare CAPTCHA. 22 | """ 23 | 24 | import sys 25 | import io 26 | import gzip 27 | import http.client 28 | import collections 29 | import logging 30 | 31 | import util 32 | 33 | log = logging.getLogger(__name__) 34 | 35 | destinations = [("www.cloudflare.com", 443)] 36 | DOMAIN, PORT = destinations[0] 37 | 38 | CAPTCHA_SIGN = b"Attention Required! | Cloudflare" 39 | 40 | # Mimic Tor Browser's request headers, so CloudFlare won't return a 403 because 41 | # it thinks we are a bot. 42 | 43 | HTTP_HEADERS = [("Host", DOMAIN), 44 | ("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:52.0) " 45 | "Gecko/20100101 Firefox/52.0"), 46 | ("Accept", "text/html,application/xhtml+xml," 47 | "application/xml;q=0.9,*/*;q=0.8"), 48 | ("Accept-Language", "en-US,en;q=0.5"), 49 | ("Accept-Encoding", "gzip, deflate, br"), 50 | ("Connection", "keep-alive"), 51 | ("Upgrade-Insecure-Requests", "1")] 52 | 53 | 54 | def decompress(data): 55 | """ 56 | Decompress gzipped HTTP response. 57 | """ 58 | 59 | try: 60 | buf = io.StringIO(data) 61 | fileobj = gzip.GzipFile(fileobj=buf) 62 | data = fileobj.read() 63 | except Exception: 64 | pass 65 | 66 | return data 67 | 68 | 69 | def is_cloudflared(exit_fpr): 70 | """ 71 | Check if site returns a CloudFlare CAPTCHA. 72 | """ 73 | 74 | exit_url = util.exiturl(exit_fpr) 75 | log.debug("Probing exit relay \"%s\"." % exit_url) 76 | 77 | conn = http.client.HTTPSConnection(DOMAIN, PORT) 78 | conn.request("GET", "/", headers=collections.OrderedDict(HTTP_HEADERS)) 79 | try: 80 | response = conn.getresponse() 81 | except Exception as err: 82 | log.warning("urlopen() over %s says: %s" % (exit_url, err)) 83 | return 84 | 85 | data = decompress(response.read()) 86 | if not data: 87 | log.warning("Did not get any data over %s." % exit_url) 88 | return 89 | 90 | if data and (CAPTCHA_SIGN in data): 91 | log.info("Exit %s sees a CAPTCHA." % exit_url) 92 | else: 93 | log.info("Exit %s does not see a CAPTCHA." % exit_url) 94 | 95 | 96 | def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs): 97 | """ 98 | Check if exit relay sees a CloudFlare CAPTCHA. 99 | """ 100 | 101 | run_python_over_tor(is_cloudflared, exit_desc.fingerprint) 102 | 103 | 104 | if __name__ == "__main__": 105 | is_cloudflared("bogus-fingerprint") 106 | sys.exit(0) 107 | -------------------------------------------------------------------------------- /src/modules/dnspoison.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2013-2017 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | 20 | """ 21 | Module to detect malfunctioning DNS resolution. 22 | """ 23 | 24 | import logging 25 | 26 | import torsocks 27 | import socket 28 | import error 29 | from util import exiturl 30 | 31 | import dns.resolver 32 | 33 | log = logging.getLogger(__name__) 34 | 35 | destinations = None 36 | domains = { 37 | "www.youporn.com": [], 38 | "youporn.com": [], 39 | "www.torproject.org": [], 40 | "www.i2p2.de": [], 41 | "torrentfreak.com": [], 42 | "blockchain.info": [], 43 | } 44 | 45 | 46 | def setup(): 47 | """ 48 | Populate the `domains' dictionary by asking our system DNS resolver. 49 | """ 50 | 51 | log.debug("Populating domain dictionary.") 52 | 53 | for domain in list(domains.keys()): 54 | response = dns.resolver.query(domain) 55 | for record in response: 56 | log.debug("Domain %s maps to %s." % (domain, record.address)) 57 | domains[domain].append(record.address) 58 | 59 | log.info("Domain whitelist: %s" % str(domains)) 60 | 61 | 62 | def resolve(exit_desc, domain, whitelist): 63 | """ 64 | Resolve a `domain' and compare it to the `whitelist'. 65 | 66 | If the domain is not part of the whitelist, an error is logged. 67 | """ 68 | 69 | exit = exiturl(exit_desc.fingerprint) 70 | sock = torsocks.torsocket() 71 | sock.settimeout(10) 72 | 73 | # Resolve the domain using Tor's SOCKS extension. 74 | 75 | try: 76 | ipv4 = sock.resolve(domain) 77 | except error.SOCKSv5Error as err: 78 | log.debug("Exit relay %s could not resolve IPv4 address for " 79 | "\"%s\" because: %s" % (exit, domain, err)) 80 | return 81 | except socket.timeout as err: 82 | log.debug("Socket over exit relay %s timed out: %s" % (exit, err)) 83 | return 84 | except EOFError as err: 85 | log.debug("EOF error: %s" % err) 86 | return 87 | 88 | if ipv4 not in whitelist: 89 | log.critical("Exit relay %s returned unexpected IPv4 address %s " 90 | "for domain %s" % (exit, ipv4, domain)) 91 | else: 92 | log.debug("IPv4 address of domain %s as expected for %s." % 93 | (domain, exit)) 94 | 95 | 96 | def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs): 97 | """ 98 | Probe the given exit relay and check if all domains resolve as expected. 99 | """ 100 | 101 | for domain in list(domains.keys()): 102 | run_python_over_tor(resolve, exit_desc, domain, domains[domain]) 103 | 104 | 105 | if __name__ == "__main__": 106 | log.critical("Module can only be run over Tor, and not stand-alone.") 107 | -------------------------------------------------------------------------------- /src/modules/dnssec.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2016 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | 20 | """ 21 | Detect exit relays whose resolver does not validate DNSSEC. 22 | """ 23 | 24 | import sys 25 | import logging 26 | import socket 27 | 28 | import error 29 | import util 30 | import torsocks 31 | 32 | log = logging.getLogger(__name__) 33 | 34 | destinations = None 35 | 36 | # The following is a deliberately broken DNSSEC domain. If we are able to 37 | # resolve it, it means that our resolver does not validate DNSSEC. 38 | 39 | BROKEN_DOMAIN = "www.dnssec-failed.org" 40 | 41 | 42 | def test_dnssec(exit_fpr): 43 | """ 44 | Test if broken DNSSEC domain can be resolved. 45 | """ 46 | 47 | exit_url = util.exiturl(exit_fpr) 48 | sock = torsocks.torsocket() 49 | sock.settimeout(10) 50 | 51 | # Resolve domain using Tor's SOCKS extension. 52 | 53 | try: 54 | ipv4 = sock.resolve(BROKEN_DOMAIN) 55 | except error.SOCKSv5Error as err: 56 | log.debug("%s did not resolve broken domain because: %s. Good." % 57 | (exit_url, err)) 58 | return 59 | except socket.timeout as err: 60 | log.debug("Socket over exit relay %s timed out: %s" % (exit_url, err)) 61 | return 62 | except Exception as err: 63 | log.debug("Could not resolve domain because: %s" % err) 64 | return 65 | 66 | log.critical("%s resolved domain to %s" % (exit_url, ipv4)) 67 | 68 | 69 | def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs): 70 | """ 71 | Test if exit relay can resolve broken domain. 72 | """ 73 | 74 | run_python_over_tor(test_dnssec, exit_desc.fingerprint) 75 | 76 | 77 | if __name__ == "__main__": 78 | log.critical("Module can only be run over Tor, not stand-alone.") 79 | sys.exit(1) 80 | -------------------------------------------------------------------------------- /src/modules/patchingCheck.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2014-2016 Philipp Winter 4 | # Copyright 2014 Josh Pitts 5 | # 6 | # This file is part of exitmap. 7 | # 8 | # exitmap is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 3 of the License, or 11 | # (at your option) any later version. 12 | # 13 | # exitmap is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with exitmap. If not, see . 20 | 21 | """ 22 | patchingCheck.py 23 | by Joshua Pitts josh.pitts@leviathansecurity.com 24 | twitter: @midnite_runr 25 | 26 | Module to detect binary patching. 27 | 28 | -USAGE- 29 | Make appropriate changes in the EDIT ME SECTION 30 | 31 | Then run: 32 | ./bin/exitmap -d 5 patchingCheck 33 | 34 | """ 35 | 36 | import sys 37 | import os 38 | try: 39 | import urllib.request, urllib.error, urllib.parse 40 | except ImportError: 41 | import urllib.request as urllib2 42 | import tempfile 43 | import logging 44 | import hashlib 45 | 46 | import util 47 | 48 | import stem.descriptor.server_descriptor as descriptor 49 | 50 | log = logging.getLogger(__name__) 51 | 52 | ####################### 53 | # EDIT ME SECTION START 54 | ####################### 55 | 56 | # EDIT ME: exitmap needs this variable to figure out which 57 | # relays can exit to the given destination(s). 58 | 59 | destinations = [("live.sysinternals.com", 80)] 60 | 61 | # Only test one binary at a time 62 | # Must provide a Download link 63 | check_files = { 64 | "http://live.sysinternals.com/psexec.exe": [None, None], 65 | # "http://www.ntcore.com/files/ExplorerSuite.exe": [None, None], 66 | } 67 | 68 | # Set UserAgent 69 | # Reference: http://www.useragentstring.com/pages/Internet%20Explorer/ 70 | test_agent = 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)' 71 | 72 | ####################### 73 | # EDIT ME SECTION END 74 | ####################### 75 | 76 | 77 | def setup(): 78 | """ 79 | Perform one-off setup tasks, i.e., download reference files. 80 | """ 81 | 82 | log.info("Creating temporary reference files.") 83 | 84 | for url, _ in check_files.items(): 85 | 86 | log.debug("Attempting to download <%s>." % url) 87 | 88 | request = urllib.request.Request(url) 89 | request.add_header('User-Agent', test_agent) 90 | 91 | try: 92 | data = urllib.request.urlopen(request).read() 93 | except Exception as err: 94 | log.warning("urlopen() failed: %s" % err) 95 | 96 | file_name = url.split("/")[-1] 97 | _, tmp_file = tempfile.mkstemp(prefix="exitmap_%s_" % file_name) 98 | 99 | with open(tmp_file, "wb") as fd: 100 | fd.write(data) 101 | 102 | log.debug("Wrote file to \"%s\"." % tmp_file) 103 | 104 | check_files[url] = [tmp_file, sha512_file(tmp_file)] 105 | 106 | 107 | def teardown(): 108 | """ 109 | Perform one-off teardown tasks, i.e., remove reference files. 110 | """ 111 | 112 | log.info("Removing reference files.") 113 | 114 | for _, file_info in check_files.items(): 115 | 116 | orig_file, _ = file_info 117 | log.info("Removing file \"%s\"." % orig_file) 118 | os.remove(orig_file) 119 | 120 | 121 | def sha512_file(file_name): 122 | """ 123 | Calculate SHA512 over the given file. 124 | """ 125 | 126 | hash_func = hashlib.sha256() 127 | 128 | with open(file_name, "rb") as fd: 129 | hash_func.update(fd.read()) 130 | 131 | return hash_func.hexdigest() 132 | 133 | 134 | def files_identical(observed_file, original_file): 135 | """ 136 | Return True if the files are identical and False otherwise. 137 | 138 | This check is necessary because sometimes file transfers are terminated 139 | before they are finished and we are left with an incomplete file. 140 | """ 141 | 142 | observed_length = os.path.getsize(observed_file) 143 | original_length = os.path.getsize(original_file) 144 | 145 | if observed_length >= original_length: 146 | return False 147 | 148 | with open(original_file) as fd: 149 | original_data = fd.read(observed_length) 150 | 151 | with open(observed_file) as fd: 152 | observed_data = fd.read() 153 | 154 | return original_data == observed_data 155 | 156 | 157 | def run_check(exit_desc): 158 | """ 159 | Download file and check if its checksum is as expected. 160 | """ 161 | 162 | exiturl = util.exiturl(exit_desc.fingerprint) 163 | 164 | for url, file_info in check_files.items(): 165 | 166 | orig_file, orig_digest = file_info 167 | 168 | log.debug("Attempting to download <%s> over %s." % (url, exiturl)) 169 | 170 | data = None 171 | 172 | request = urllib.request.Request(url) 173 | request.add_header('User-Agent', test_agent) 174 | 175 | try: 176 | data = urllib.request.urlopen(request, timeout=20).read() 177 | except Exception as err: 178 | log.warning("urlopen() failed for %s: %s" % (exiturl, err)) 179 | continue 180 | 181 | if not data: 182 | log.warning("No data received from <%s> over %s." % (url, exiturl)) 183 | continue 184 | 185 | file_name = url.split("/")[-1] 186 | _, tmp_file = tempfile.mkstemp(prefix="exitmap_%s_%s_" % 187 | (exit_desc.fingerprint, file_name)) 188 | 189 | with open(tmp_file, "wb") as fd: 190 | fd.write(data) 191 | 192 | observed_digest = sha512_file(tmp_file) 193 | 194 | if (observed_digest != orig_digest) and \ 195 | (not files_identical(tmp_file, orig_file)): 196 | 197 | log.critical("File \"%s\" differs from reference file \"%s\". " 198 | "Downloaded over exit relay %s." % 199 | (tmp_file, orig_file, exiturl)) 200 | 201 | else: 202 | log.debug("File \"%s\" fetched over %s as expected." % 203 | (tmp_file, exiturl)) 204 | 205 | os.remove(tmp_file) 206 | 207 | 208 | def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs): 209 | """ 210 | Probe the given exit relay and look for modified binaries. 211 | """ 212 | 213 | run_python_over_tor(run_check, exit_desc) 214 | 215 | 216 | def main(): 217 | """ 218 | Entry point when invoked over the command line. 219 | """ 220 | 221 | setup() 222 | 223 | desc = descriptor.ServerDescriptor("") 224 | desc.fingerprint = "bogus" 225 | run_check(desc) 226 | 227 | teardown() 228 | 229 | return 0 230 | 231 | 232 | if __name__ == "__main__": 233 | sys.exit(main()) 234 | -------------------------------------------------------------------------------- /src/modules/rtt.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Copyright 2013-2016 Philipp Winter 4 | # Copyright 2016 Zack Weinberg 5 | # 6 | # This file is part of exitmap. 7 | # 8 | # exitmap is free software: you can redistribute it and/or modify 9 | # it under the terms of the GNU General Public License as published by 10 | # the Free Software Foundation, either version 3 of the License, or 11 | # (at your option) any later version. 12 | # 13 | # exitmap is distributed in the hope that it will be useful, 14 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 | # GNU General Public License for more details. 17 | # 18 | # You should have received a copy of the GNU General Public License 19 | # along with exitmap. If not, see . 20 | 21 | """ 22 | Module to measure round-trip times through an exit to various 23 | destinations. Each destination will receive ten TCP connections from 24 | each scanned exit, no faster than one connection every 50ms. The module 25 | doesn't care whether it gets a SYN/ACK or a RST in response -- either 26 | way, the round-trip time is recorded and the connection is dropped. 27 | 28 | Connections are attempted to one of port 53, 22, 443 or 80, depending 29 | on what's allowed by the exit's policy. 30 | 31 | Until modules can take command-line arguments, the destinations should 32 | be specified in a text file named "rtt-destinations.txt", one IP 33 | address per line. (You _may_ use hostnames, but if you do, they will 34 | be resolved directly, not via Tor.) 35 | """ 36 | 37 | import sys 38 | import os 39 | import logging 40 | import csv 41 | import errno 42 | import random 43 | import socket 44 | import util 45 | 46 | # We don't _need_ the top-level exitmap module, but this is the most 47 | # reliable way to figure out whether we need to add the directory with 48 | # the utility modules that we _do_ need to sys.path. 49 | try: 50 | import exitmap 51 | except ImportError: 52 | current_path = os.path.dirname(__file__) 53 | src_path = os.path.abspath(os.path.join(current_path, "..")) 54 | sys.path.insert(0, src_path) 55 | import exitmap 56 | 57 | try: 58 | from time import monotonic as tick 59 | except ImportError: 60 | # FIXME: Maybe use ctypes to get at clock_gettime(CLOCK_MONOTONIC)? 61 | from time import time as tick 62 | 63 | try: 64 | import selectors 65 | except ImportError: 66 | import selectors34 as selectors 67 | 68 | # Configuration parameters: 69 | # The set of ports that we consider connecting to. 70 | PREFERRED_PORT_ORDER = (53, 22, 443, 80) 71 | 72 | # The total number of connections to make to each host. 73 | CONNECTIONS_PER_HOST = 10 74 | 75 | # The number of hosts to connect to in parallel. Note that we will 76 | # _not_ connect to any one host more than once at a time. 77 | PARALLEL_CONNECTIONS = 4 78 | 79 | # The delay between successive connections (seconds) 80 | CONNECTION_SPACING = 0.25 81 | 82 | # The per-connection timeout (seconds). 83 | CONNECTION_TIMEOUT = 10.0 84 | 85 | 86 | log = logging.getLogger(__name__) 87 | 88 | 89 | def progress(total, pending, complete): 90 | log.info("{:>6}/{:>6} complete, {} pending" 91 | .format(complete, total, pending)) 92 | 93 | 94 | def perform_probes(addresses, spacing, parallel, timeout, wr): 95 | """Make a TCP connection to each of the ADDRESSES, in order, and 96 | measure the time for connect(2) to either succeed or fail -- we 97 | don't care which. Each element of the iterable ADDRESSES should 98 | be an AF_INET address 2-tuple (i.e. ('a.b.c.d', n)). Successive 99 | connections will be no closer to each other in time than SPACING 100 | floating-point seconds. No more than PARALLEL concurrent 101 | connections will occur at any one time. Sockets that have neither 102 | succeeded nor failed to connect after TIMEOUT floating-point 103 | seconds will be treated as having failed. No data is transmitted; 104 | each socket is closed immediately after the connection resolves. 105 | 106 | The results are written to the csv.writer object WR; each row of the 107 | file will be ,,. 108 | """ 109 | 110 | if timeout <= 0: 111 | raise ValueError("timeout must be positive") 112 | if spacing <= 0: 113 | raise ValueError("spacing must be positive") 114 | if parallel < 1: 115 | raise ValueError("parallel must be at least 1") 116 | 117 | sel = selectors.DefaultSelector() 118 | EVENT_READ = selectors.EVENT_READ 119 | AF_INET = socket.AF_INET 120 | SOCK_STREAM = socket.SOCK_STREAM 121 | 122 | EINPROGRESS = errno.EINPROGRESS 123 | CONN_RESOLVED = (0, 124 | errno.ECONNREFUSED, 125 | errno.EHOSTUNREACH, 126 | errno.ENETUNREACH, 127 | errno.ETIMEDOUT, 128 | errno.ECONNRESET) 129 | 130 | pending = set() 131 | addresses.reverse() 132 | last_connection = 0 133 | last_progress = 0 134 | total = len(addresses) 135 | complete = 0 136 | change = False 137 | 138 | try: 139 | while pending or addresses: 140 | now = tick() 141 | if change or now - last_progress > 10: 142 | progress(total, len(pending), complete) 143 | last_progress = now 144 | change = False 145 | 146 | if (len(pending) < parallel and addresses 147 | and now - last_connection >= spacing): 148 | 149 | addr = addresses.pop() 150 | sock = socket.socket(AF_INET, SOCK_STREAM) 151 | sock.setblocking(False) 152 | 153 | last_connection = tick() 154 | err = sock.connect_ex(addr) 155 | log.debug("Socket %d connecting to %r returned %d/%s", 156 | sock.fileno(), addr, err, os.strerror(err)) 157 | if err == EINPROGRESS: 158 | # This is the expected case: the connection attempt is 159 | # in progress and we must wait for results. 160 | pending.add(sel.register(sock, EVENT_READ, 161 | (addr, last_connection))) 162 | change = True 163 | 164 | elif err in CONN_RESOLVED: 165 | # The connection attempt resolved before connect() 166 | # returned. 167 | after = tick() 168 | sock.close() 169 | wr.writerow((addr[0], addr[1], after - now)) 170 | complete += 1 171 | change = True 172 | 173 | else: 174 | # Something dire has happened and we probably 175 | # can't continue (for instance, there's no local 176 | # network connection). 177 | exc = socket.error(err, os.strerror(err)) 178 | exc.filename = '%s:%d' % addr 179 | raise exc 180 | 181 | events = sel.select(spacing) 182 | after = tick() 183 | # We don't care whether each connection succeeded or failed. 184 | for key, _ in events: 185 | addr, before = key.data 186 | sock = key.fileobj 187 | log.debug("Socket %d connecting to %r resolved", 188 | sock.fileno(), addr) 189 | 190 | sel.unregister(sock) 191 | sock.close() 192 | pending.remove(key) 193 | wr.writerow((addr[0], addr[1], after - before)) 194 | complete += 1 195 | change = True 196 | 197 | # Check for timeouts. 198 | for key in list(pending): 199 | addr, before = key.data 200 | if after - before >= timeout: 201 | sock = key.fileobj 202 | log.debug("Socket %d connecting to %r timed out", 203 | sock.fileno(), addr) 204 | sel.unregister(sock) 205 | sock.close() 206 | pending.remove(key) 207 | wr.writerow((addr[0], addr[1], after - before)) 208 | complete += 1 209 | change = True 210 | 211 | # end while 212 | progress(total, len(pending), complete) 213 | 214 | finally: 215 | for key in pending: 216 | sel.unregister(key.fileobj) 217 | key.fileobj.close() 218 | sel.close() 219 | 220 | 221 | def choose_probe_order(dests): 222 | """Choose a randomized probe order for the destinations DESTS, which is 223 | a set of (host, port) pairs. The return value is a list acceptable 224 | as the ADDRESSES argument to perform_probes.""" 225 | 226 | hosts = {} 227 | for h, p in dests: 228 | if h not in hosts: hosts[h] = set() 229 | hosts[h].add(p) 230 | 231 | remaining = {} 232 | last_appearance = {} 233 | full_address = {} 234 | for host, usable_ports in hosts.items(): 235 | for p in PREFERRED_PORT_ORDER: 236 | if p in usable_ports: 237 | full_address[host] = (host, p) 238 | remaining[host] = CONNECTIONS_PER_HOST 239 | last_appearance[host] = -1 240 | 241 | rv = [] 242 | deadcycles = 0 243 | while remaining: 244 | ks = list(remaining.keys()) 245 | x = random.choice(ks) 246 | last = last_appearance[x] 247 | if last == -1 or (len(rv) - last) >= (len(ks) // 4): 248 | last_appearance[x] = len(rv) 249 | rv.append(full_address[x]) 250 | remaining[x] -= 1 251 | if not remaining[x]: 252 | del remaining[x] 253 | deadcycles = 0 254 | else: 255 | deadcycles += 1 256 | if deadcycles == 10: 257 | raise RuntimeError("choose_probe_order: 10 dead cycles\n" 258 | "remaining: %r\n" 259 | "last_appearance: %r\n" 260 | % (remaining, last_appearance)) 261 | return rv 262 | 263 | 264 | def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, 265 | destinations, **kwargs): 266 | """ 267 | Probe the given exit relay. 268 | """ 269 | addresses = choose_probe_order(destinations) 270 | 271 | try: 272 | os.makedirs(util.analysis_dir) 273 | except OSError as err: 274 | if err.errno != errno.EEXIST: 275 | raise 276 | 277 | with open(os.path.join(util.analysis_dir, 278 | exit_desc.fingerprint + ".csv"), "wt") as f: 279 | wr = csv.writer(f, quoting=csv.QUOTE_MINIMAL, lineterminator='\n') 280 | wr.writerow(("host", "port", "elapsed")) 281 | 282 | run_python_over_tor(perform_probes, 283 | addresses, 284 | CONNECTION_SPACING, 285 | PARALLEL_CONNECTIONS, 286 | CONNECTION_TIMEOUT, 287 | wr) 288 | 289 | # exitmap needs this variable to figure out which relays can exit to the given 290 | # destination(s). 291 | 292 | destinations = None 293 | 294 | 295 | def setup(): 296 | ds = set() 297 | with open("rtt-destinations.txt") as f: 298 | for line in f: 299 | line = line.strip() 300 | if not line or line[0] == '#': continue 301 | ipaddr = socket.getaddrinfo( 302 | line, 80, socket.AF_INET, socket.SOCK_STREAM, 0, 0)[0][4][0] 303 | 304 | for p in PREFERRED_PORT_ORDER: 305 | ds.add((ipaddr, p)) 306 | 307 | global destinations 308 | destinations = sorted(ds) 309 | -------------------------------------------------------------------------------- /src/modules/testfds.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | # Copyright 2014-2016 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | 20 | """ 21 | This module attempts to fetch a simple web page. If this succeeds, we know 22 | that the relay (probably) has enough file descriptors. 23 | """ 24 | 25 | import sys 26 | import re 27 | import logging 28 | import urllib.request, urllib.error, urllib.parse 29 | 30 | from util import exiturl 31 | 32 | import stem.descriptor.server_descriptor as descriptor 33 | import socks 34 | 35 | log = logging.getLogger(__name__) 36 | 37 | destinations = [("people.torproject.org", 443)] 38 | 39 | 40 | def fetch_page(exit_desc): 41 | 42 | expected = "This file is to check if your exit relay has enough file " \ 43 | "descriptors to fetch it." 44 | 45 | exit_url = exiturl(exit_desc.fingerprint) 46 | 47 | log.debug("Probing exit relay %s." % exit_url) 48 | 49 | data = None 50 | try: 51 | data = urllib.request.urlopen("https://people.torproject.org/~phw/check_file", 52 | timeout=10).read().decode("utf-8") 53 | except Exception as err: 54 | log.warning("urllib.request.urlopen for %s says: %s." % 55 | (exit_desc.fingerprint, err)) 56 | return 57 | 58 | if not data: 59 | log.warning("Exit relay %s did not return data." % exit_url) 60 | return 61 | 62 | data = data.strip() 63 | 64 | if not re.match(expected, data): 65 | log.warning("Got unexpected response from %s: %s." % (exit_url, data)) 66 | else: 67 | log.debug("Exit relay %s worked fine." % exit_url) 68 | 69 | 70 | def probe(exit_desc, run_python_over_tor, run_cmd_over_tor, **kwargs): 71 | """ 72 | Attempts to fetch a small web page and yells if this fails. 73 | """ 74 | 75 | run_python_over_tor(fetch_page, exit_desc) 76 | 77 | 78 | def main(): 79 | """ 80 | Entry point when invoked over the command line. 81 | """ 82 | 83 | desc = descriptor.ServerDescriptor("") 84 | desc.fingerprint = "bogus" 85 | fetch_page(desc) 86 | 87 | return 0 88 | 89 | if __name__ == "__main__": 90 | sys.exit(main()) 91 | -------------------------------------------------------------------------------- /src/relayselector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | # Copyright 2013-2017 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | 20 | """ 21 | Extracts exit relays with given attributes from consensus. 22 | """ 23 | 24 | import os 25 | import sys 26 | import argparse 27 | import logging 28 | 29 | import stem 30 | import stem.descriptor 31 | 32 | import util 33 | 34 | log = logging.getLogger(__name__) 35 | 36 | 37 | def parse_cmd_args(): 38 | """ 39 | Parses and returns command line arguments. 40 | """ 41 | 42 | parser = argparse.ArgumentParser(description="%s selects a subset of Tor " 43 | "exit relays." % sys.argv[0]) 44 | 45 | parser.add_argument("-b", "--badexit", action="store_true", default=None, 46 | help="Select bad exit relays.") 47 | 48 | parser.add_argument("-g", "--goodexit", action="store_true", default=None, 49 | help="Select non-bad exit relays.") 50 | 51 | parser.add_argument("-c", "--countrycode", type=str, default=None, 52 | help="Two-letter country code to select.") 53 | 54 | parser.add_argument("data_dir", metavar="DATA_DIR", type=str, default=None, 55 | help="Tor's data directory.") 56 | 57 | parser.add_argument("-v", "--version", type=str, default=None, 58 | help="Show relays with a specific version.") 59 | 60 | parser.add_argument("-n", "--nickname", type=str, default=None, 61 | help="Select relay with the given nickname.") 62 | 63 | parser.add_argument("-a", "--address", type=str, default=None, 64 | help="Select relays which contain the given (part " 65 | "of an) IPv4 address.") 66 | 67 | return parser.parse_args() 68 | 69 | 70 | def get_fingerprints(cached_consensus_path, exclude=[]): 71 | """ 72 | Get all relay fingerprints in the provided consensus. 73 | 74 | Relay fingerprints which are present in the list `exclude' are ignored. 75 | """ 76 | 77 | fingerprints = [] 78 | 79 | for desc in stem.descriptor.parse_file(cached_consensus_path): 80 | if desc.fingerprint not in exclude: 81 | fingerprints.append(desc.fingerprint) 82 | 83 | return fingerprints 84 | 85 | 86 | def get_exit_policies(cached_descriptors_path): 87 | """Read all relays' full exit policies from "cached_descriptors".""" 88 | 89 | try: 90 | have_exit_policy = {} 91 | 92 | # We don't validate to work around the following issue: 93 | # 94 | for desc in stem.descriptor.parse_file(cached_descriptors_path, 95 | validate=False): 96 | if desc.exit_policy.is_exiting_allowed(): 97 | have_exit_policy[desc.fingerprint] = desc 98 | 99 | return have_exit_policy 100 | 101 | except IOError as err: 102 | log.critical("File \"%s\" could not be read: %s" % 103 | (cached_descriptors_path, err)) 104 | sys.exit(1) 105 | 106 | 107 | def get_cached_consensus(cached_consensus_path): 108 | """Read relays' summarized descriptors from "cached_consensus".""" 109 | try: 110 | cached_consensus = {} 111 | for desc in stem.descriptor.parse_file(cached_consensus_path): 112 | cached_consensus[desc.fingerprint] = desc 113 | return cached_consensus 114 | 115 | except IOError as err: 116 | log.critical("File \"%s\" could not be read: %s" % 117 | (cached_consensus_path, err)) 118 | sys.exit(1) 119 | 120 | 121 | def get_exits(data_dir, 122 | good_exit=True, bad_exit=False, 123 | version=None, nickname=None, address=None, country_code=None, 124 | requested_exits=None, destinations=None): 125 | """Load the Tor network consensus from DATA_DIR, and extract all exit 126 | relays that have the desired set of attributes. Specifically: 127 | 128 | - requested_exits: If not None, must be a list of fingerprints, 129 | and only those relays will be included in the results. 130 | 131 | - country_code, version, nickname, address: 132 | If not None, only relays with the specified attributes 133 | will be included in the results. 134 | 135 | - bad_exit, good_exit: If True, the respective type of exit will 136 | be included. At least one should be True, or else the results 137 | will be empty. 138 | 139 | These combine as follows: 140 | 141 | exit.fingerprint IN requested_exits 142 | AND exit.country_code == country_code 143 | AND exit.version == version 144 | AND exit.nickname IN nickname 145 | AND exit.address IN address 146 | AND ( (bad_exit AND exit.is_bad_exit) 147 | OR (good_exit AND NOT exit.is_bad_exit)) 148 | 149 | In all cases, the criterion is skipped if the argument is None. 150 | 151 | Finally, 'destinations' is considered. If this is None, all 152 | results from the above filter expression are returned. Otherwise, 153 | 'destinations' must be a set of (host, port) pairs, and only exits 154 | that will connect to *some* of these destinations will be included 155 | in the results. 156 | 157 | Returns a dictionary, whose keys are the selected relays' fingerprints. 158 | The value for each fingerprint is a set of (host, port) pairs that 159 | that exit is willing to connect to; this is always a subset of the 160 | input 'destinations' set. (If 'destinations' was None, each value 161 | is a pseudo-set object for which '(host, port) in s' always 162 | returns True.) 163 | """ 164 | 165 | 166 | cached_consensus_path = os.path.join(data_dir, "cached-consensus") 167 | cached_descriptors_path = os.path.join(data_dir, "cached-descriptors") 168 | 169 | cached_consensus = get_cached_consensus(cached_consensus_path) 170 | have_exit_policy = get_exit_policies(cached_descriptors_path) 171 | 172 | # Drop all exit relays which have a descriptor, but either did not 173 | # make it into the consensus at all, or are not marked as exits there. 174 | class StubDesc(object): 175 | def __init__(self): 176 | self.flags = frozenset() 177 | stub_desc = StubDesc() 178 | 179 | exit_candidates = [ 180 | desc 181 | for fpr, desc in have_exit_policy.items() 182 | if stem.Flag.EXIT in cached_consensus.get(fpr, stub_desc).flags 183 | ] 184 | 185 | log.info("In addition to %d exit relays, %d relays have non-empty exit " 186 | "policy but no exit flag.", len(exit_candidates), 187 | len(have_exit_policy) - len(exit_candidates)) 188 | if not exit_candidates: 189 | log.warning("No relays have both a non-empty exit policy and an exit " 190 | "flag. This probably means the cached network consensus " 191 | "is invalid.") 192 | return {} 193 | 194 | if bad_exit and good_exit: 195 | pass # All exits are either bad or good. 196 | elif bad_exit: 197 | exit_candidates = [ 198 | desc for desc in exit_candidates 199 | if stem.Flag.BADEXIT in cached_consensus[desc.fingerprint].flags 200 | ] 201 | if not exit_candidates: 202 | log.warning("There are no bad exits in the current consensus.") 203 | return {} 204 | elif good_exit: 205 | exit_candidates = [ 206 | desc for desc in exit_candidates 207 | if stem.Flag.BADEXIT not in cached_consensus[desc.fingerprint].flags 208 | ] 209 | if not exit_candidates: 210 | log.warning("There are no good exits in the current consensus.") 211 | return {} 212 | else: 213 | # This was probably a programming error. 214 | log.warning("get_exits() called with bad_exits=False and " 215 | "good_exits=False; this always returns zero exits") 216 | return {} 217 | 218 | # Filter conditions are checked from cheapest to most expensive. 219 | if address or nickname or version or requested_exits: 220 | exit_candidates = [ 221 | desc for desc in exit_candidates 222 | if ((not address or address in desc.address) and 223 | (not nickname or nickname in desc.nickname) and 224 | (not version or version == str(desc.tor_version)) and 225 | (not requested_exits or desc.fingerprint in requested_exits)) 226 | ] 227 | if not exit_candidates: 228 | log.warning("No exit relays meet basic filter conditions.") 229 | return {} 230 | 231 | if country_code: 232 | try: 233 | relay_fprs = frozenset(util.get_relays_in_country(country_code)) 234 | except Exception as err: 235 | log.warning("get_relays_in_country() failed: %s" % err) 236 | relay_fprs = [] 237 | 238 | exit_candidates = [ 239 | desc for desc in exit_candidates 240 | if desc.fingerprint in relay_fprs 241 | ] 242 | if not exit_candidates: 243 | log.warning("No exit relays meet country-code filter condition.") 244 | return {} 245 | 246 | if not destinations: 247 | class UniversalSet(object): 248 | """A universal set contains everything, but cannot be enumerated. 249 | 250 | If the caller of get_exits does not specify destinations, 251 | its return value maps all fingerprints to a universal set, 252 | so that it can still fulfill the contract of returning a 253 | dictionary of the form { fingerprint : set(...) }. 254 | """ 255 | def __nonzero__(self): return True 256 | 257 | def __contains__(self, obj): return True 258 | 259 | # __len__ is obliged to return a positive integer. 260 | def __len__(self): return sys.maxsize 261 | us = UniversalSet() 262 | exit_destinations = { 263 | desc.fingerprint: us for desc in exit_candidates} 264 | else: 265 | exit_destinations = {} 266 | for desc in exit_candidates: 267 | policy = have_exit_policy[desc.fingerprint].exit_policy 268 | ok_dests = frozenset(d for d in destinations 269 | if policy.can_exit_to(*d)) 270 | if ok_dests: 271 | exit_destinations[desc.fingerprint] = ok_dests 272 | 273 | log.info("%d out of %d exit relays meet all filter conditions." 274 | % (len(exit_destinations), len(have_exit_policy))) 275 | return exit_destinations 276 | 277 | 278 | def main(): 279 | args = parse_cmd_args() 280 | 281 | exits = get_exits(args.data_dir, 282 | country_code = args.countrycode, 283 | bad_exit = args.badexit, 284 | good_exit = args.goodexit, 285 | version = args.version, 286 | nickname = args.nickname, 287 | address = args.address) 288 | for e in exits.keys(): 289 | print("https://atlas.torproject.org/#details/%s" % e) 290 | 291 | 292 | if __name__ == "__main__": 293 | sys.exit(main()) 294 | -------------------------------------------------------------------------------- /src/selectors34.py: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | """Selectors module. 3 | 4 | This module allows high-level and efficient I/O multiplexing, built upon the 5 | `select` module primitives. 6 | 7 | Python 2 backport by Charles-François Natali and Victor Stinner: 8 | https://pypi.python.org/pypi/selectors34 9 | 10 | """ 11 | 12 | from abc import ABCMeta, abstractmethod 13 | from collections import namedtuple, Mapping 14 | import math 15 | import select 16 | import sys 17 | 18 | import six 19 | 20 | # compatibility code 21 | PY33 = (sys.version_info >= (3, 3)) 22 | 23 | 24 | def _wrap_error(exc, mapping, key): 25 | if key not in mapping: 26 | return 27 | new_err_cls = mapping[key] 28 | new_err = new_err_cls(*exc.args) 29 | 30 | # raise a new exception with the original traceback 31 | if hasattr(exc, '__traceback__'): 32 | traceback = exc.__traceback__ 33 | else: 34 | traceback = sys.exc_info()[2] 35 | six.reraise(new_err_cls, new_err, traceback) 36 | 37 | 38 | if PY33: 39 | import builtins 40 | 41 | BlockingIOError = builtins.BlockingIOError 42 | BrokenPipeError = builtins.BrokenPipeError 43 | ChildProcessError = builtins.ChildProcessError 44 | ConnectionRefusedError = builtins.ConnectionRefusedError 45 | ConnectionResetError = builtins.ConnectionResetError 46 | InterruptedError = builtins.InterruptedError 47 | ConnectionAbortedError = builtins.ConnectionAbortedError 48 | PermissionError = builtins.PermissionError 49 | FileNotFoundError = builtins.FileNotFoundError 50 | ProcessLookupError = builtins.ProcessLookupError 51 | 52 | def wrap_error(func, *args, **kw): 53 | return func(*args, **kw) 54 | else: 55 | import errno 56 | import select 57 | import socket 58 | 59 | class BlockingIOError(OSError): 60 | pass 61 | 62 | class BrokenPipeError(OSError): 63 | pass 64 | 65 | class ChildProcessError(OSError): 66 | pass 67 | 68 | class ConnectionRefusedError(OSError): 69 | pass 70 | 71 | class InterruptedError(OSError): 72 | pass 73 | 74 | class ConnectionResetError(OSError): 75 | pass 76 | 77 | class ConnectionAbortedError(OSError): 78 | pass 79 | 80 | class PermissionError(OSError): 81 | pass 82 | 83 | class FileNotFoundError(OSError): 84 | pass 85 | 86 | class ProcessLookupError(OSError): 87 | pass 88 | 89 | _MAP_ERRNO = { 90 | errno.EACCES: PermissionError, 91 | errno.EAGAIN: BlockingIOError, 92 | errno.EALREADY: BlockingIOError, 93 | errno.ECHILD: ChildProcessError, 94 | errno.ECONNABORTED: ConnectionAbortedError, 95 | errno.ECONNREFUSED: ConnectionRefusedError, 96 | errno.ECONNRESET: ConnectionResetError, 97 | errno.EINPROGRESS: BlockingIOError, 98 | errno.EINTR: InterruptedError, 99 | errno.ENOENT: FileNotFoundError, 100 | errno.EPERM: PermissionError, 101 | errno.EPIPE: BrokenPipeError, 102 | errno.ESHUTDOWN: BrokenPipeError, 103 | errno.EWOULDBLOCK: BlockingIOError, 104 | errno.ESRCH: ProcessLookupError, 105 | } 106 | 107 | def wrap_error(func, *args, **kw): 108 | """ 109 | Wrap socket.error, IOError, OSError, select.error to raise new specialized 110 | exceptions of Python 3.3 like InterruptedError (PEP 3151). 111 | """ 112 | try: 113 | return func(*args, **kw) 114 | except (socket.error, IOError, OSError) as exc: 115 | if hasattr(exc, 'winerror'): 116 | _wrap_error(exc, _MAP_ERRNO, exc.winerror) 117 | # _MAP_ERRNO does not contain all Windows errors. 118 | # For some errors like "file not found", exc.errno should 119 | # be used (ex: ENOENT). 120 | _wrap_error(exc, _MAP_ERRNO, exc.errno) 121 | raise 122 | except select.error as exc: 123 | if exc.args: 124 | _wrap_error(exc, _MAP_ERRNO, exc.args[0]) 125 | raise 126 | 127 | # generic events, that must be mapped to implementation-specific ones 128 | EVENT_READ = (1 << 0) 129 | EVENT_WRITE = (1 << 1) 130 | 131 | 132 | def _fileobj_to_fd(fileobj): 133 | """Return a file descriptor from a file object. 134 | 135 | Parameters: 136 | fileobj -- file object or file descriptor 137 | 138 | Returns: 139 | corresponding file descriptor 140 | 141 | Raises: 142 | ValueError if the object is invalid 143 | """ 144 | if isinstance(fileobj, six.integer_types): 145 | fd = fileobj 146 | else: 147 | try: 148 | fd = int(fileobj.fileno()) 149 | except (AttributeError, TypeError, ValueError): 150 | raise ValueError("Invalid file object: " 151 | "{0!r}".format(fileobj)) 152 | if fd < 0: 153 | raise ValueError("Invalid file descriptor: {0}".format(fd)) 154 | return fd 155 | 156 | 157 | SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) 158 | """Object used to associate a file object to its backing file descriptor, 159 | selected event mask and attached data.""" 160 | 161 | 162 | class _SelectorMapping(Mapping): 163 | """Mapping of file objects to selector keys.""" 164 | 165 | def __init__(self, selector): 166 | self._selector = selector 167 | 168 | def __len__(self): 169 | return len(self._selector._fd_to_key) 170 | 171 | def __getitem__(self, fileobj): 172 | try: 173 | fd = self._selector._fileobj_lookup(fileobj) 174 | return self._selector._fd_to_key[fd] 175 | except KeyError: 176 | raise KeyError("{0!r} is not registered".format(fileobj)) 177 | 178 | def __iter__(self): 179 | return iter(self._selector._fd_to_key) 180 | 181 | 182 | class BaseSelector(six.with_metaclass(ABCMeta)): 183 | """Selector abstract base class. 184 | 185 | A selector supports registering file objects to be monitored for specific 186 | I/O events. 187 | 188 | A file object is a file descriptor or any object with a `fileno()` method. 189 | An arbitrary object can be attached to the file object, which can be used 190 | for example to store context information, a callback, etc. 191 | 192 | A selector can use various implementations (select(), poll(), epoll()...) 193 | depending on the platform. The default `Selector` class uses the most 194 | efficient implementation on the current platform. 195 | """ 196 | 197 | @abstractmethod 198 | def register(self, fileobj, events, data=None): 199 | """Register a file object. 200 | 201 | Parameters: 202 | fileobj -- file object or file descriptor 203 | events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) 204 | data -- attached data 205 | 206 | Returns: 207 | SelectorKey instance 208 | 209 | Raises: 210 | ValueError if events is invalid 211 | KeyError if fileobj is already registered 212 | OSError if fileobj is closed or otherwise is unacceptable to 213 | the underlying system call (if a system call is made) 214 | 215 | Note: 216 | OSError may or may not be raised 217 | """ 218 | raise NotImplementedError 219 | 220 | @abstractmethod 221 | def unregister(self, fileobj): 222 | """Unregister a file object. 223 | 224 | Parameters: 225 | fileobj -- file object or file descriptor 226 | 227 | Returns: 228 | SelectorKey instance 229 | 230 | Raises: 231 | KeyError if fileobj is not registered 232 | 233 | Note: 234 | If fileobj is registered but has since been closed this does 235 | *not* raise OSError (even if the wrapped syscall does) 236 | """ 237 | raise NotImplementedError 238 | 239 | def modify(self, fileobj, events, data=None): 240 | """Change a registered file object monitored events or attached data. 241 | 242 | Parameters: 243 | fileobj -- file object or file descriptor 244 | events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) 245 | data -- attached data 246 | 247 | Returns: 248 | SelectorKey instance 249 | 250 | Raises: 251 | Anything that unregister() or register() raises 252 | """ 253 | self.unregister(fileobj) 254 | return self.register(fileobj, events, data) 255 | 256 | @abstractmethod 257 | def select(self, timeout=None): 258 | """Perform the actual selection, until some monitored file objects are 259 | ready or a timeout expires. 260 | 261 | Parameters: 262 | timeout -- if timeout > 0, this specifies the maximum wait time, in 263 | seconds 264 | if timeout <= 0, the select() call won't block, and will 265 | report the currently ready file objects 266 | if timeout is None, select() will block until a monitored 267 | file object becomes ready 268 | 269 | Returns: 270 | list of (key, events) for ready file objects 271 | `events` is a bitwise mask of EVENT_READ|EVENT_WRITE 272 | """ 273 | raise NotImplementedError 274 | 275 | def close(self): 276 | """Close the selector. 277 | 278 | This must be called to make sure that any underlying resource is freed. 279 | """ 280 | pass 281 | 282 | def get_key(self, fileobj): 283 | """Return the key associated to a registered file object. 284 | 285 | Returns: 286 | SelectorKey for this file object 287 | """ 288 | mapping = self.get_map() 289 | if mapping is None: 290 | raise RuntimeError('Selector is closed') 291 | try: 292 | return mapping[fileobj] 293 | except KeyError: 294 | raise KeyError("{0!r} is not registered".format(fileobj)) 295 | 296 | @abstractmethod 297 | def get_map(self): 298 | """Return a mapping of file objects to selector keys.""" 299 | raise NotImplementedError 300 | 301 | def __enter__(self): 302 | return self 303 | 304 | def __exit__(self, *args): 305 | self.close() 306 | 307 | 308 | class _BaseSelectorImpl(BaseSelector): 309 | """Base selector implementation.""" 310 | 311 | def __init__(self): 312 | # this maps file descriptors to keys 313 | self._fd_to_key = {} 314 | # read-only mapping returned by get_map() 315 | self._map = _SelectorMapping(self) 316 | 317 | def _fileobj_lookup(self, fileobj): 318 | """Return a file descriptor from a file object. 319 | 320 | This wraps _fileobj_to_fd() to do an exhaustive search in case 321 | the object is invalid but we still have it in our map. This 322 | is used by unregister() so we can unregister an object that 323 | was previously registered even if it is closed. It is also 324 | used by _SelectorMapping. 325 | """ 326 | try: 327 | return _fileobj_to_fd(fileobj) 328 | except ValueError: 329 | # Do an exhaustive search. 330 | for key in self._fd_to_key.values(): 331 | if key.fileobj is fileobj: 332 | return key.fd 333 | # Raise ValueError after all. 334 | raise 335 | 336 | def register(self, fileobj, events, data=None): 337 | if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): 338 | raise ValueError("Invalid events: {0!r}".format(events)) 339 | 340 | key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) 341 | 342 | if key.fd in self._fd_to_key: 343 | raise KeyError("{0!r} (FD {1}) is already registered" 344 | .format(fileobj, key.fd)) 345 | 346 | self._fd_to_key[key.fd] = key 347 | return key 348 | 349 | def unregister(self, fileobj): 350 | try: 351 | key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) 352 | except KeyError: 353 | raise KeyError("{0!r} is not registered".format(fileobj)) 354 | return key 355 | 356 | def modify(self, fileobj, events, data=None): 357 | # TODO: Subclasses can probably optimize this even further. 358 | try: 359 | key = self._fd_to_key[self._fileobj_lookup(fileobj)] 360 | except KeyError: 361 | raise KeyError("{0!r} is not registered".format(fileobj)) 362 | if events != key.events: 363 | self.unregister(fileobj) 364 | key = self.register(fileobj, events, data) 365 | elif data != key.data: 366 | # Use a shortcut to update the data. 367 | key = key._replace(data=data) 368 | self._fd_to_key[key.fd] = key 369 | return key 370 | 371 | def close(self): 372 | self._fd_to_key.clear() 373 | self._map = None 374 | 375 | def get_map(self): 376 | return self._map 377 | 378 | def _key_from_fd(self, fd): 379 | """Return the key associated to a given file descriptor. 380 | 381 | Parameters: 382 | fd -- file descriptor 383 | 384 | Returns: 385 | corresponding key, or None if not found 386 | """ 387 | try: 388 | return self._fd_to_key[fd] 389 | except KeyError: 390 | return None 391 | 392 | 393 | class SelectSelector(_BaseSelectorImpl): 394 | """Select-based selector.""" 395 | 396 | def __init__(self): 397 | super(SelectSelector, self).__init__() 398 | self._readers = set() 399 | self._writers = set() 400 | 401 | def register(self, fileobj, events, data=None): 402 | key = super(SelectSelector, self).register(fileobj, events, data) 403 | if events & EVENT_READ: 404 | self._readers.add(key.fd) 405 | if events & EVENT_WRITE: 406 | self._writers.add(key.fd) 407 | return key 408 | 409 | def unregister(self, fileobj): 410 | key = super(SelectSelector, self).unregister(fileobj) 411 | self._readers.discard(key.fd) 412 | self._writers.discard(key.fd) 413 | return key 414 | 415 | if sys.platform == 'win32': 416 | def _select(self, r, w, _, timeout=None): 417 | r, w, x = select.select(r, w, w, timeout) 418 | return r, w + x, [] 419 | else: 420 | _select = select.select 421 | 422 | def select(self, timeout=None): 423 | timeout = None if timeout is None else max(timeout, 0) 424 | ready = [] 425 | try: 426 | r, w, _ = wrap_error(self._select, 427 | self._readers, self._writers, [], timeout) 428 | except InterruptedError: 429 | return ready 430 | r = set(r) 431 | w = set(w) 432 | for fd in r | w: 433 | events = 0 434 | if fd in r: 435 | events |= EVENT_READ 436 | if fd in w: 437 | events |= EVENT_WRITE 438 | 439 | key = self._key_from_fd(fd) 440 | if key: 441 | ready.append((key, events & key.events)) 442 | return ready 443 | 444 | 445 | if hasattr(select, 'poll'): 446 | 447 | class PollSelector(_BaseSelectorImpl): 448 | """Poll-based selector.""" 449 | 450 | def __init__(self): 451 | super(PollSelector, self).__init__() 452 | self._poll = select.poll() 453 | 454 | def register(self, fileobj, events, data=None): 455 | key = super(PollSelector, self).register(fileobj, events, data) 456 | poll_events = 0 457 | if events & EVENT_READ: 458 | poll_events |= select.POLLIN 459 | if events & EVENT_WRITE: 460 | poll_events |= select.POLLOUT 461 | self._poll.register(key.fd, poll_events) 462 | return key 463 | 464 | def unregister(self, fileobj): 465 | key = super(PollSelector, self).unregister(fileobj) 466 | self._poll.unregister(key.fd) 467 | return key 468 | 469 | def select(self, timeout=None): 470 | if timeout is None: 471 | timeout = None 472 | elif timeout <= 0: 473 | timeout = 0 474 | else: 475 | # poll() has a resolution of 1 millisecond, round away from 476 | # zero to wait *at least* timeout seconds. 477 | timeout = int(math.ceil(timeout * 1e3)) 478 | ready = [] 479 | try: 480 | fd_event_list = wrap_error(self._poll.poll, timeout) 481 | except InterruptedError: 482 | return ready 483 | for fd, event in fd_event_list: 484 | events = 0 485 | if event & ~select.POLLIN: 486 | events |= EVENT_WRITE 487 | if event & ~select.POLLOUT: 488 | events |= EVENT_READ 489 | 490 | key = self._key_from_fd(fd) 491 | if key: 492 | ready.append((key, events & key.events)) 493 | return ready 494 | 495 | 496 | if hasattr(select, 'epoll'): 497 | 498 | class EpollSelector(_BaseSelectorImpl): 499 | """Epoll-based selector.""" 500 | 501 | def __init__(self): 502 | super(EpollSelector, self).__init__() 503 | self._epoll = select.epoll() 504 | 505 | def fileno(self): 506 | return self._epoll.fileno() 507 | 508 | def register(self, fileobj, events, data=None): 509 | key = super(EpollSelector, self).register(fileobj, events, data) 510 | epoll_events = 0 511 | if events & EVENT_READ: 512 | epoll_events |= select.EPOLLIN 513 | if events & EVENT_WRITE: 514 | epoll_events |= select.EPOLLOUT 515 | self._epoll.register(key.fd, epoll_events) 516 | return key 517 | 518 | def unregister(self, fileobj): 519 | key = super(EpollSelector, self).unregister(fileobj) 520 | try: 521 | self._epoll.unregister(key.fd) 522 | except IOError: 523 | # This can happen if the FD was closed since it 524 | # was registered. 525 | pass 526 | return key 527 | 528 | def select(self, timeout=None): 529 | if timeout is None: 530 | timeout = -1 531 | elif timeout <= 0: 532 | timeout = 0 533 | else: 534 | # epoll_wait() has a resolution of 1 millisecond, round away 535 | # from zero to wait *at least* timeout seconds. 536 | timeout = math.ceil(timeout * 1e3) * 1e-3 537 | 538 | # epoll_wait() expects `maxevents` to be greater than zero; 539 | # we want to make sure that `select()` can be called when no 540 | # FD is registered. 541 | max_ev = max(len(self._fd_to_key), 1) 542 | 543 | ready = [] 544 | try: 545 | fd_event_list = wrap_error(self._epoll.poll, timeout, max_ev) 546 | except InterruptedError: 547 | return ready 548 | for fd, event in fd_event_list: 549 | events = 0 550 | if event & ~select.EPOLLIN: 551 | events |= EVENT_WRITE 552 | if event & ~select.EPOLLOUT: 553 | events |= EVENT_READ 554 | 555 | key = self._key_from_fd(fd) 556 | if key: 557 | ready.append((key, events & key.events)) 558 | return ready 559 | 560 | def close(self): 561 | self._epoll.close() 562 | super(EpollSelector, self).close() 563 | 564 | 565 | if hasattr(select, 'devpoll'): 566 | 567 | class DevpollSelector(_BaseSelectorImpl): 568 | """Solaris /dev/poll selector.""" 569 | 570 | def __init__(self): 571 | super(DevpollSelector, self).__init__() 572 | self._devpoll = select.devpoll() 573 | 574 | def fileno(self): 575 | return self._devpoll.fileno() 576 | 577 | def register(self, fileobj, events, data=None): 578 | key = super(DevpollSelector, self).register(fileobj, events, data) 579 | poll_events = 0 580 | if events & EVENT_READ: 581 | poll_events |= select.POLLIN 582 | if events & EVENT_WRITE: 583 | poll_events |= select.POLLOUT 584 | self._devpoll.register(key.fd, poll_events) 585 | return key 586 | 587 | def unregister(self, fileobj): 588 | key = super(DevpollSelector, self).unregister(fileobj) 589 | self._devpoll.unregister(key.fd) 590 | return key 591 | 592 | def select(self, timeout=None): 593 | if timeout is None: 594 | timeout = None 595 | elif timeout <= 0: 596 | timeout = 0 597 | else: 598 | # devpoll() has a resolution of 1 millisecond, round away from 599 | # zero to wait *at least* timeout seconds. 600 | timeout = math.ceil(timeout * 1e3) 601 | ready = [] 602 | try: 603 | fd_event_list = self._devpoll.poll(timeout) 604 | except InterruptedError: 605 | return ready 606 | for fd, event in fd_event_list: 607 | events = 0 608 | if event & ~select.POLLIN: 609 | events |= EVENT_WRITE 610 | if event & ~select.POLLOUT: 611 | events |= EVENT_READ 612 | 613 | key = self._key_from_fd(fd) 614 | if key: 615 | ready.append((key, events & key.events)) 616 | return ready 617 | 618 | def close(self): 619 | self._devpoll.close() 620 | super(DevpollSelector, self).close() 621 | 622 | 623 | if hasattr(select, 'kqueue'): 624 | 625 | class KqueueSelector(_BaseSelectorImpl): 626 | """Kqueue-based selector.""" 627 | 628 | def __init__(self): 629 | super(KqueueSelector, self).__init__() 630 | self._kqueue = select.kqueue() 631 | 632 | def fileno(self): 633 | return self._kqueue.fileno() 634 | 635 | def register(self, fileobj, events, data=None): 636 | key = super(KqueueSelector, self).register(fileobj, events, data) 637 | if events & EVENT_READ: 638 | kev = select.kevent(key.fd, select.KQ_FILTER_READ, 639 | select.KQ_EV_ADD) 640 | self._kqueue.control([kev], 0, 0) 641 | if events & EVENT_WRITE: 642 | kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, 643 | select.KQ_EV_ADD) 644 | self._kqueue.control([kev], 0, 0) 645 | return key 646 | 647 | def unregister(self, fileobj): 648 | key = super(KqueueSelector, self).unregister(fileobj) 649 | if key.events & EVENT_READ: 650 | kev = select.kevent(key.fd, select.KQ_FILTER_READ, 651 | select.KQ_EV_DELETE) 652 | try: 653 | self._kqueue.control([kev], 0, 0) 654 | except OSError: 655 | # This can happen if the FD was closed since it 656 | # was registered. 657 | pass 658 | if key.events & EVENT_WRITE: 659 | kev = select.kevent(key.fd, select.KQ_FILTER_WRITE, 660 | select.KQ_EV_DELETE) 661 | try: 662 | self._kqueue.control([kev], 0, 0) 663 | except OSError: 664 | # See comment above. 665 | pass 666 | return key 667 | 668 | def select(self, timeout=None): 669 | timeout = None if timeout is None else max(timeout, 0) 670 | max_ev = len(self._fd_to_key) 671 | ready = [] 672 | try: 673 | kev_list = wrap_error(self._kqueue.control, 674 | None, max_ev, timeout) 675 | except InterruptedError: 676 | return ready 677 | for kev in kev_list: 678 | fd = kev.ident 679 | flag = kev.filter 680 | events = 0 681 | if flag == select.KQ_FILTER_READ: 682 | events |= EVENT_READ 683 | if flag == select.KQ_FILTER_WRITE: 684 | events |= EVENT_WRITE 685 | 686 | key = self._key_from_fd(fd) 687 | if key: 688 | ready.append((key, events & key.events)) 689 | return ready 690 | 691 | def close(self): 692 | self._kqueue.close() 693 | super(KqueueSelector, self).close() 694 | 695 | 696 | # Choose the best implementation, roughly: 697 | # epoll|kqueue|devpoll > poll > select. 698 | # select() also can't accept a FD > FD_SETSIZE (usually around 1024) 699 | if 'KqueueSelector' in globals(): 700 | DefaultSelector = KqueueSelector 701 | elif 'EpollSelector' in globals(): 702 | DefaultSelector = EpollSelector 703 | elif 'DevpollSelector' in globals(): 704 | DefaultSelector = DevpollSelector 705 | elif 'PollSelector' in globals(): 706 | DefaultSelector = PollSelector 707 | else: 708 | DefaultSelector = SelectSelector 709 | -------------------------------------------------------------------------------- /src/six.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2010-2016 Benjamin Peterson 2 | # 3 | # Permission is hereby granted, free of charge, to any person obtaining a copy 4 | # of this software and associated documentation files (the "Software"), to deal 5 | # in the Software without restriction, including without limitation the rights 6 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | # copies of the Software, and to permit persons to whom the Software is 8 | # furnished to do so, subject to the following conditions: 9 | # 10 | # The above copyright notice and this permission notice shall be included in 11 | # all copies or substantial portions of the Software. 12 | # 13 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | # SOFTWARE. 20 | 21 | """Utilities for writing code that runs on Python 2 and 3""" 22 | 23 | from __future__ import absolute_import 24 | 25 | import functools 26 | import itertools 27 | import operator 28 | import sys 29 | import types 30 | 31 | __author__ = "Benjamin Peterson " 32 | __version__ = "1.10.0" 33 | 34 | 35 | # Useful for very coarse version differentiation. 36 | PY2 = sys.version_info[0] == 2 37 | PY3 = sys.version_info[0] == 3 38 | PY34 = sys.version_info[0:2] >= (3, 4) 39 | 40 | if PY3: 41 | string_types = str, 42 | integer_types = int, 43 | class_types = type, 44 | text_type = str 45 | binary_type = bytes 46 | 47 | MAXSIZE = sys.maxsize 48 | else: 49 | string_types = basestring, 50 | integer_types = (int, long) 51 | class_types = (type, types.ClassType) 52 | text_type = unicode 53 | binary_type = str 54 | 55 | if sys.platform.startswith("java"): 56 | # Jython always uses 32 bits. 57 | MAXSIZE = int((1 << 31) - 1) 58 | else: 59 | # It's possible to have sizeof(long) != sizeof(Py_ssize_t). 60 | class X(object): 61 | 62 | def __len__(self): 63 | return 1 << 31 64 | try: 65 | len(X()) 66 | except OverflowError: 67 | # 32-bit 68 | MAXSIZE = int((1 << 31) - 1) 69 | else: 70 | # 64-bit 71 | MAXSIZE = int((1 << 63) - 1) 72 | del X 73 | 74 | 75 | def _add_doc(func, doc): 76 | """Add documentation to a function.""" 77 | func.__doc__ = doc 78 | 79 | 80 | def _import_module(name): 81 | """Import module, returning the module after the last dot.""" 82 | __import__(name) 83 | return sys.modules[name] 84 | 85 | 86 | class _LazyDescr(object): 87 | 88 | def __init__(self, name): 89 | self.name = name 90 | 91 | def __get__(self, obj, tp): 92 | result = self._resolve() 93 | setattr(obj, self.name, result) # Invokes __set__. 94 | try: 95 | # This is a bit ugly, but it avoids running this again by 96 | # removing this descriptor. 97 | delattr(obj.__class__, self.name) 98 | except AttributeError: 99 | pass 100 | return result 101 | 102 | 103 | class MovedModule(_LazyDescr): 104 | 105 | def __init__(self, name, old, new=None): 106 | super(MovedModule, self).__init__(name) 107 | if PY3: 108 | if new is None: 109 | new = name 110 | self.mod = new 111 | else: 112 | self.mod = old 113 | 114 | def _resolve(self): 115 | return _import_module(self.mod) 116 | 117 | def __getattr__(self, attr): 118 | _module = self._resolve() 119 | value = getattr(_module, attr) 120 | setattr(self, attr, value) 121 | return value 122 | 123 | 124 | class _LazyModule(types.ModuleType): 125 | 126 | def __init__(self, name): 127 | super(_LazyModule, self).__init__(name) 128 | self.__doc__ = self.__class__.__doc__ 129 | 130 | def __dir__(self): 131 | attrs = ["__doc__", "__name__"] 132 | attrs += [attr.name for attr in self._moved_attributes] 133 | return attrs 134 | 135 | # Subclasses should override this 136 | _moved_attributes = [] 137 | 138 | 139 | class MovedAttribute(_LazyDescr): 140 | 141 | def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): 142 | super(MovedAttribute, self).__init__(name) 143 | if PY3: 144 | if new_mod is None: 145 | new_mod = name 146 | self.mod = new_mod 147 | if new_attr is None: 148 | if old_attr is None: 149 | new_attr = name 150 | else: 151 | new_attr = old_attr 152 | self.attr = new_attr 153 | else: 154 | self.mod = old_mod 155 | if old_attr is None: 156 | old_attr = name 157 | self.attr = old_attr 158 | 159 | def _resolve(self): 160 | module = _import_module(self.mod) 161 | return getattr(module, self.attr) 162 | 163 | 164 | class _SixMetaPathImporter(object): 165 | 166 | """ 167 | A meta path importer to import six.moves and its submodules. 168 | 169 | This class implements a PEP302 finder and loader. It should be compatible 170 | with Python 2.5 and all existing versions of Python3 171 | """ 172 | 173 | def __init__(self, six_module_name): 174 | self.name = six_module_name 175 | self.known_modules = {} 176 | 177 | def _add_module(self, mod, *fullnames): 178 | for fullname in fullnames: 179 | self.known_modules[self.name + "." + fullname] = mod 180 | 181 | def _get_module(self, fullname): 182 | return self.known_modules[self.name + "." + fullname] 183 | 184 | def find_module(self, fullname, path=None): 185 | if fullname in self.known_modules: 186 | return self 187 | return None 188 | 189 | def __get_module(self, fullname): 190 | try: 191 | return self.known_modules[fullname] 192 | except KeyError: 193 | raise ImportError("This loader does not know module " + fullname) 194 | 195 | def load_module(self, fullname): 196 | try: 197 | # in case of a reload 198 | return sys.modules[fullname] 199 | except KeyError: 200 | pass 201 | mod = self.__get_module(fullname) 202 | if isinstance(mod, MovedModule): 203 | mod = mod._resolve() 204 | else: 205 | mod.__loader__ = self 206 | sys.modules[fullname] = mod 207 | return mod 208 | 209 | def is_package(self, fullname): 210 | """ 211 | Return true, if the named module is a package. 212 | 213 | We need this method to get correct spec objects with 214 | Python 3.4 (see PEP451) 215 | """ 216 | return hasattr(self.__get_module(fullname), "__path__") 217 | 218 | def get_code(self, fullname): 219 | """Return None 220 | 221 | Required, if is_package is implemented""" 222 | self.__get_module(fullname) # eventually raises ImportError 223 | return None 224 | get_source = get_code # same as get_code 225 | 226 | _importer = _SixMetaPathImporter(__name__) 227 | 228 | 229 | class _MovedItems(_LazyModule): 230 | 231 | """Lazy loading of moved objects""" 232 | __path__ = [] # mark as package 233 | 234 | 235 | _moved_attributes = [ 236 | MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), 237 | MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), 238 | MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), 239 | MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), 240 | MovedAttribute("intern", "__builtin__", "sys"), 241 | MovedAttribute("map", "itertools", "builtins", "imap", "map"), 242 | MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), 243 | MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), 244 | MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), 245 | MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), 246 | MovedAttribute("reduce", "__builtin__", "functools"), 247 | MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), 248 | MovedAttribute("StringIO", "StringIO", "io"), 249 | MovedAttribute("UserDict", "UserDict", "collections"), 250 | MovedAttribute("UserList", "UserList", "collections"), 251 | MovedAttribute("UserString", "UserString", "collections"), 252 | MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), 253 | MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), 254 | MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), 255 | MovedModule("builtins", "__builtin__"), 256 | MovedModule("configparser", "ConfigParser"), 257 | MovedModule("copyreg", "copy_reg"), 258 | MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), 259 | MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), 260 | MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), 261 | MovedModule("http_cookies", "Cookie", "http.cookies"), 262 | MovedModule("html_entities", "htmlentitydefs", "html.entities"), 263 | MovedModule("html_parser", "HTMLParser", "html.parser"), 264 | MovedModule("http_client", "httplib", "http.client"), 265 | MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), 266 | MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), 267 | MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), 268 | MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), 269 | MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), 270 | MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), 271 | MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), 272 | MovedModule("cPickle", "cPickle", "pickle"), 273 | MovedModule("queue", "Queue"), 274 | MovedModule("reprlib", "repr"), 275 | MovedModule("socketserver", "SocketServer"), 276 | MovedModule("_thread", "thread", "_thread"), 277 | MovedModule("tkinter", "Tkinter"), 278 | MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), 279 | MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), 280 | MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), 281 | MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), 282 | MovedModule("tkinter_tix", "Tix", "tkinter.tix"), 283 | MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), 284 | MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), 285 | MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), 286 | MovedModule("tkinter_colorchooser", "tkColorChooser", 287 | "tkinter.colorchooser"), 288 | MovedModule("tkinter_commondialog", "tkCommonDialog", 289 | "tkinter.commondialog"), 290 | MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), 291 | MovedModule("tkinter_font", "tkFont", "tkinter.font"), 292 | MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), 293 | MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", 294 | "tkinter.simpledialog"), 295 | MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), 296 | MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), 297 | MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), 298 | MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), 299 | MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), 300 | MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), 301 | ] 302 | # Add windows specific modules. 303 | if sys.platform == "win32": 304 | _moved_attributes += [ 305 | MovedModule("winreg", "_winreg"), 306 | ] 307 | 308 | for attr in _moved_attributes: 309 | setattr(_MovedItems, attr.name, attr) 310 | if isinstance(attr, MovedModule): 311 | _importer._add_module(attr, "moves." + attr.name) 312 | del attr 313 | 314 | _MovedItems._moved_attributes = _moved_attributes 315 | 316 | moves = _MovedItems(__name__ + ".moves") 317 | _importer._add_module(moves, "moves") 318 | 319 | 320 | class Module_six_moves_urllib_parse(_LazyModule): 321 | 322 | """Lazy loading of moved objects in six.moves.urllib_parse""" 323 | 324 | 325 | _urllib_parse_moved_attributes = [ 326 | MovedAttribute("ParseResult", "urlparse", "urllib.parse"), 327 | MovedAttribute("SplitResult", "urlparse", "urllib.parse"), 328 | MovedAttribute("parse_qs", "urlparse", "urllib.parse"), 329 | MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), 330 | MovedAttribute("urldefrag", "urlparse", "urllib.parse"), 331 | MovedAttribute("urljoin", "urlparse", "urllib.parse"), 332 | MovedAttribute("urlparse", "urlparse", "urllib.parse"), 333 | MovedAttribute("urlsplit", "urlparse", "urllib.parse"), 334 | MovedAttribute("urlunparse", "urlparse", "urllib.parse"), 335 | MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), 336 | MovedAttribute("quote", "urllib", "urllib.parse"), 337 | MovedAttribute("quote_plus", "urllib", "urllib.parse"), 338 | MovedAttribute("unquote", "urllib", "urllib.parse"), 339 | MovedAttribute("unquote_plus", "urllib", "urllib.parse"), 340 | MovedAttribute("urlencode", "urllib", "urllib.parse"), 341 | MovedAttribute("splitquery", "urllib", "urllib.parse"), 342 | MovedAttribute("splittag", "urllib", "urllib.parse"), 343 | MovedAttribute("splituser", "urllib", "urllib.parse"), 344 | MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), 345 | MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), 346 | MovedAttribute("uses_params", "urlparse", "urllib.parse"), 347 | MovedAttribute("uses_query", "urlparse", "urllib.parse"), 348 | MovedAttribute("uses_relative", "urlparse", "urllib.parse"), 349 | ] 350 | for attr in _urllib_parse_moved_attributes: 351 | setattr(Module_six_moves_urllib_parse, attr.name, attr) 352 | del attr 353 | 354 | Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes 355 | 356 | _importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), 357 | "moves.urllib_parse", "moves.urllib.parse") 358 | 359 | 360 | class Module_six_moves_urllib_error(_LazyModule): 361 | 362 | """Lazy loading of moved objects in six.moves.urllib_error""" 363 | 364 | 365 | _urllib_error_moved_attributes = [ 366 | MovedAttribute("URLError", "urllib2", "urllib.error"), 367 | MovedAttribute("HTTPError", "urllib2", "urllib.error"), 368 | MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), 369 | ] 370 | for attr in _urllib_error_moved_attributes: 371 | setattr(Module_six_moves_urllib_error, attr.name, attr) 372 | del attr 373 | 374 | Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes 375 | 376 | _importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), 377 | "moves.urllib_error", "moves.urllib.error") 378 | 379 | 380 | class Module_six_moves_urllib_request(_LazyModule): 381 | 382 | """Lazy loading of moved objects in six.moves.urllib_request""" 383 | 384 | 385 | _urllib_request_moved_attributes = [ 386 | MovedAttribute("urlopen", "urllib2", "urllib.request"), 387 | MovedAttribute("install_opener", "urllib2", "urllib.request"), 388 | MovedAttribute("build_opener", "urllib2", "urllib.request"), 389 | MovedAttribute("pathname2url", "urllib", "urllib.request"), 390 | MovedAttribute("url2pathname", "urllib", "urllib.request"), 391 | MovedAttribute("getproxies", "urllib", "urllib.request"), 392 | MovedAttribute("Request", "urllib2", "urllib.request"), 393 | MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), 394 | MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), 395 | MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), 396 | MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), 397 | MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), 398 | MovedAttribute("BaseHandler", "urllib2", "urllib.request"), 399 | MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), 400 | MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), 401 | MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), 402 | MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), 403 | MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), 404 | MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), 405 | MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), 406 | MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), 407 | MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), 408 | MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), 409 | MovedAttribute("FileHandler", "urllib2", "urllib.request"), 410 | MovedAttribute("FTPHandler", "urllib2", "urllib.request"), 411 | MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), 412 | MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), 413 | MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), 414 | MovedAttribute("urlretrieve", "urllib", "urllib.request"), 415 | MovedAttribute("urlcleanup", "urllib", "urllib.request"), 416 | MovedAttribute("URLopener", "urllib", "urllib.request"), 417 | MovedAttribute("FancyURLopener", "urllib", "urllib.request"), 418 | MovedAttribute("proxy_bypass", "urllib", "urllib.request"), 419 | ] 420 | for attr in _urllib_request_moved_attributes: 421 | setattr(Module_six_moves_urllib_request, attr.name, attr) 422 | del attr 423 | 424 | Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes 425 | 426 | _importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), 427 | "moves.urllib_request", "moves.urllib.request") 428 | 429 | 430 | class Module_six_moves_urllib_response(_LazyModule): 431 | 432 | """Lazy loading of moved objects in six.moves.urllib_response""" 433 | 434 | 435 | _urllib_response_moved_attributes = [ 436 | MovedAttribute("addbase", "urllib", "urllib.response"), 437 | MovedAttribute("addclosehook", "urllib", "urllib.response"), 438 | MovedAttribute("addinfo", "urllib", "urllib.response"), 439 | MovedAttribute("addinfourl", "urllib", "urllib.response"), 440 | ] 441 | for attr in _urllib_response_moved_attributes: 442 | setattr(Module_six_moves_urllib_response, attr.name, attr) 443 | del attr 444 | 445 | Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes 446 | 447 | _importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), 448 | "moves.urllib_response", "moves.urllib.response") 449 | 450 | 451 | class Module_six_moves_urllib_robotparser(_LazyModule): 452 | 453 | """Lazy loading of moved objects in six.moves.urllib_robotparser""" 454 | 455 | 456 | _urllib_robotparser_moved_attributes = [ 457 | MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), 458 | ] 459 | for attr in _urllib_robotparser_moved_attributes: 460 | setattr(Module_six_moves_urllib_robotparser, attr.name, attr) 461 | del attr 462 | 463 | Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes 464 | 465 | _importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), 466 | "moves.urllib_robotparser", "moves.urllib.robotparser") 467 | 468 | 469 | class Module_six_moves_urllib(types.ModuleType): 470 | 471 | """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" 472 | __path__ = [] # mark as package 473 | parse = _importer._get_module("moves.urllib_parse") 474 | error = _importer._get_module("moves.urllib_error") 475 | request = _importer._get_module("moves.urllib_request") 476 | response = _importer._get_module("moves.urllib_response") 477 | robotparser = _importer._get_module("moves.urllib_robotparser") 478 | 479 | def __dir__(self): 480 | return ['parse', 'error', 'request', 'response', 'robotparser'] 481 | 482 | _importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), 483 | "moves.urllib") 484 | 485 | 486 | def add_move(move): 487 | """Add an item to six.moves.""" 488 | setattr(_MovedItems, move.name, move) 489 | 490 | 491 | def remove_move(name): 492 | """Remove item from six.moves.""" 493 | try: 494 | delattr(_MovedItems, name) 495 | except AttributeError: 496 | try: 497 | del moves.__dict__[name] 498 | except KeyError: 499 | raise AttributeError("no such move, %r" % (name,)) 500 | 501 | 502 | if PY3: 503 | _meth_func = "__func__" 504 | _meth_self = "__self__" 505 | 506 | _func_closure = "__closure__" 507 | _func_code = "__code__" 508 | _func_defaults = "__defaults__" 509 | _func_globals = "__globals__" 510 | else: 511 | _meth_func = "im_func" 512 | _meth_self = "im_self" 513 | 514 | _func_closure = "func_closure" 515 | _func_code = "func_code" 516 | _func_defaults = "func_defaults" 517 | _func_globals = "func_globals" 518 | 519 | 520 | try: 521 | advance_iterator = next 522 | except NameError: 523 | def advance_iterator(it): 524 | return it.next() 525 | next = advance_iterator 526 | 527 | 528 | try: 529 | callable = callable 530 | except NameError: 531 | def callable(obj): 532 | return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) 533 | 534 | 535 | if PY3: 536 | def get_unbound_function(unbound): 537 | return unbound 538 | 539 | create_bound_method = types.MethodType 540 | 541 | def create_unbound_method(func, cls): 542 | return func 543 | 544 | Iterator = object 545 | else: 546 | def get_unbound_function(unbound): 547 | return unbound.im_func 548 | 549 | def create_bound_method(func, obj): 550 | return types.MethodType(func, obj, obj.__class__) 551 | 552 | def create_unbound_method(func, cls): 553 | return types.MethodType(func, None, cls) 554 | 555 | class Iterator(object): 556 | 557 | def next(self): 558 | return type(self).__next__(self) 559 | 560 | callable = callable 561 | _add_doc(get_unbound_function, 562 | """Get the function out of a possibly unbound function""") 563 | 564 | 565 | get_method_function = operator.attrgetter(_meth_func) 566 | get_method_self = operator.attrgetter(_meth_self) 567 | get_function_closure = operator.attrgetter(_func_closure) 568 | get_function_code = operator.attrgetter(_func_code) 569 | get_function_defaults = operator.attrgetter(_func_defaults) 570 | get_function_globals = operator.attrgetter(_func_globals) 571 | 572 | 573 | if PY3: 574 | def iterkeys(d, **kw): 575 | return iter(d.keys(**kw)) 576 | 577 | def itervalues(d, **kw): 578 | return iter(d.values(**kw)) 579 | 580 | def iteritems(d, **kw): 581 | return iter(d.items(**kw)) 582 | 583 | def iterlists(d, **kw): 584 | return iter(d.lists(**kw)) 585 | 586 | viewkeys = operator.methodcaller("keys") 587 | 588 | viewvalues = operator.methodcaller("values") 589 | 590 | viewitems = operator.methodcaller("items") 591 | else: 592 | def iterkeys(d, **kw): 593 | return d.iterkeys(**kw) 594 | 595 | def itervalues(d, **kw): 596 | return d.itervalues(**kw) 597 | 598 | def iteritems(d, **kw): 599 | return d.iteritems(**kw) 600 | 601 | def iterlists(d, **kw): 602 | return d.iterlists(**kw) 603 | 604 | viewkeys = operator.methodcaller("viewkeys") 605 | 606 | viewvalues = operator.methodcaller("viewvalues") 607 | 608 | viewitems = operator.methodcaller("viewitems") 609 | 610 | _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") 611 | _add_doc(itervalues, "Return an iterator over the values of a dictionary.") 612 | _add_doc(iteritems, 613 | "Return an iterator over the (key, value) pairs of a dictionary.") 614 | _add_doc(iterlists, 615 | "Return an iterator over the (key, [values]) pairs of a dictionary.") 616 | 617 | 618 | if PY3: 619 | def b(s): 620 | return s.encode("latin-1") 621 | 622 | def u(s): 623 | return s 624 | unichr = chr 625 | import struct 626 | int2byte = struct.Struct(">B").pack 627 | del struct 628 | byte2int = operator.itemgetter(0) 629 | indexbytes = operator.getitem 630 | iterbytes = iter 631 | import io 632 | StringIO = io.StringIO 633 | BytesIO = io.BytesIO 634 | _assertCountEqual = "assertCountEqual" 635 | if sys.version_info[1] <= 1: 636 | _assertRaisesRegex = "assertRaisesRegexp" 637 | _assertRegex = "assertRegexpMatches" 638 | else: 639 | _assertRaisesRegex = "assertRaisesRegex" 640 | _assertRegex = "assertRegex" 641 | else: 642 | def b(s): 643 | return s 644 | # Workaround for standalone backslash 645 | 646 | def u(s): 647 | return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") 648 | unichr = unichr 649 | int2byte = chr 650 | 651 | def byte2int(bs): 652 | return ord(bs[0]) 653 | 654 | def indexbytes(buf, i): 655 | return ord(buf[i]) 656 | iterbytes = functools.partial(itertools.imap, ord) 657 | import StringIO 658 | StringIO = BytesIO = StringIO.StringIO 659 | _assertCountEqual = "assertItemsEqual" 660 | _assertRaisesRegex = "assertRaisesRegexp" 661 | _assertRegex = "assertRegexpMatches" 662 | _add_doc(b, """Byte literal""") 663 | _add_doc(u, """Text literal""") 664 | 665 | 666 | def assertCountEqual(self, *args, **kwargs): 667 | return getattr(self, _assertCountEqual)(*args, **kwargs) 668 | 669 | 670 | def assertRaisesRegex(self, *args, **kwargs): 671 | return getattr(self, _assertRaisesRegex)(*args, **kwargs) 672 | 673 | 674 | def assertRegex(self, *args, **kwargs): 675 | return getattr(self, _assertRegex)(*args, **kwargs) 676 | 677 | 678 | if PY3: 679 | exec_ = getattr(moves.builtins, "exec") 680 | 681 | def reraise(tp, value, tb=None): 682 | if value is None: 683 | value = tp() 684 | if value.__traceback__ is not tb: 685 | raise value.with_traceback(tb) 686 | raise value 687 | 688 | else: 689 | def exec_(_code_, _globs_=None, _locs_=None): 690 | """Execute code in a namespace.""" 691 | if _globs_ is None: 692 | frame = sys._getframe(1) 693 | _globs_ = frame.f_globals 694 | if _locs_ is None: 695 | _locs_ = frame.f_locals 696 | del frame 697 | elif _locs_ is None: 698 | _locs_ = _globs_ 699 | exec("""exec _code_ in _globs_, _locs_""") 700 | 701 | exec_("""def reraise(tp, value, tb=None): 702 | raise tp, value, tb 703 | """) 704 | 705 | 706 | if sys.version_info[:2] == (3, 2): 707 | exec_("""def raise_from(value, from_value): 708 | if from_value is None: 709 | raise value 710 | raise value from from_value 711 | """) 712 | elif sys.version_info[:2] > (3, 2): 713 | exec_("""def raise_from(value, from_value): 714 | raise value from from_value 715 | """) 716 | else: 717 | def raise_from(value, from_value): 718 | raise value 719 | 720 | 721 | print_ = getattr(moves.builtins, "print", None) 722 | if print_ is None: 723 | def print_(*args, **kwargs): 724 | """The new-style print function for Python 2.4 and 2.5.""" 725 | fp = kwargs.pop("file", sys.stdout) 726 | if fp is None: 727 | return 728 | 729 | def write(data): 730 | if not isinstance(data, basestring): 731 | data = str(data) 732 | # If the file has an encoding, encode unicode with it. 733 | if (isinstance(fp, file) and 734 | isinstance(data, unicode) and 735 | fp.encoding is not None): 736 | errors = getattr(fp, "errors", None) 737 | if errors is None: 738 | errors = "strict" 739 | data = data.encode(fp.encoding, errors) 740 | fp.write(data) 741 | want_unicode = False 742 | sep = kwargs.pop("sep", None) 743 | if sep is not None: 744 | if isinstance(sep, unicode): 745 | want_unicode = True 746 | elif not isinstance(sep, str): 747 | raise TypeError("sep must be None or a string") 748 | end = kwargs.pop("end", None) 749 | if end is not None: 750 | if isinstance(end, unicode): 751 | want_unicode = True 752 | elif not isinstance(end, str): 753 | raise TypeError("end must be None or a string") 754 | if kwargs: 755 | raise TypeError("invalid keyword arguments to print()") 756 | if not want_unicode: 757 | for arg in args: 758 | if isinstance(arg, unicode): 759 | want_unicode = True 760 | break 761 | if want_unicode: 762 | newline = unicode("\n") 763 | space = unicode(" ") 764 | else: 765 | newline = "\n" 766 | space = " " 767 | if sep is None: 768 | sep = space 769 | if end is None: 770 | end = newline 771 | for i, arg in enumerate(args): 772 | if i: 773 | write(sep) 774 | write(arg) 775 | write(end) 776 | if sys.version_info[:2] < (3, 3): 777 | _print = print_ 778 | 779 | def print_(*args, **kwargs): 780 | fp = kwargs.get("file", sys.stdout) 781 | flush = kwargs.pop("flush", False) 782 | _print(*args, **kwargs) 783 | if flush and fp is not None: 784 | fp.flush() 785 | 786 | _add_doc(reraise, """Reraise an exception.""") 787 | 788 | if sys.version_info[0:2] < (3, 4): 789 | def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, 790 | updated=functools.WRAPPER_UPDATES): 791 | def wrapper(f): 792 | f = functools.wraps(wrapped, assigned, updated)(f) 793 | f.__wrapped__ = wrapped 794 | return f 795 | return wrapper 796 | else: 797 | wraps = functools.wraps 798 | 799 | 800 | def with_metaclass(meta, *bases): 801 | """Create a base class with a metaclass.""" 802 | # This requires a bit of explanation: the basic idea is to make a dummy 803 | # metaclass for one level of class instantiation that replaces itself with 804 | # the actual metaclass. 805 | class metaclass(meta): 806 | 807 | def __new__(cls, name, this_bases, d): 808 | return meta(name, bases, d) 809 | return type.__new__(metaclass, 'temporary_class', (), {}) 810 | 811 | 812 | def add_metaclass(metaclass): 813 | """Class decorator for creating a class with a metaclass.""" 814 | def wrapper(cls): 815 | orig_vars = cls.__dict__.copy() 816 | slots = orig_vars.get('__slots__') 817 | if slots is not None: 818 | if isinstance(slots, str): 819 | slots = [slots] 820 | for slots_var in slots: 821 | orig_vars.pop(slots_var) 822 | orig_vars.pop('__dict__', None) 823 | orig_vars.pop('__weakref__', None) 824 | return metaclass(cls.__name__, cls.__bases__, orig_vars) 825 | return wrapper 826 | 827 | 828 | def python_2_unicode_compatible(klass): 829 | """ 830 | A decorator that defines __unicode__ and __str__ methods under Python 2. 831 | Under Python 3 it does nothing. 832 | 833 | To support Python 2 and 3 with a single code base, define a __str__ method 834 | returning text and apply this decorator to the class. 835 | """ 836 | if PY2: 837 | if '__str__' not in klass.__dict__: 838 | raise ValueError("@python_2_unicode_compatible cannot be applied " 839 | "to %s because it doesn't define __str__()." % 840 | klass.__name__) 841 | klass.__unicode__ = klass.__str__ 842 | klass.__str__ = lambda self: self.__unicode__().encode('utf-8') 843 | return klass 844 | 845 | 846 | # Complete the moves implementation. 847 | # This code is at the end of this module to speed up module loading. 848 | # Turn this module into a package. 849 | __path__ = [] # required for PEP 302 and PEP 451 850 | __package__ = __name__ # see PEP 366 @ReservedAssignment 851 | if globals().get("__spec__") is not None: 852 | __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable 853 | # Remove other six meta path importers, since they cause problems. This can 854 | # happen if six is removed from sys.modules and then reloaded. (Setuptools does 855 | # this for some reason.) 856 | if sys.meta_path: 857 | for i, importer in enumerate(sys.meta_path): 858 | # Here's some real nastiness: Another "instance" of the six module might 859 | # be floating around. Therefore, we can't use isinstance() to check for 860 | # the six meta path importer, since the other six instance will have 861 | # inserted an importer with different class. 862 | if (type(importer).__name__ == "_SixMetaPathImporter" and 863 | importer.name == __name__): 864 | del sys.meta_path[i] 865 | break 866 | del i, importer 867 | # Finally, add the importer to the meta path import hook. 868 | sys.meta_path.append(_importer) 869 | -------------------------------------------------------------------------------- /src/stats.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2016 Philipp Winter 2 | # 3 | # This file is part of exitmap. 4 | # 5 | # exitmap is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # exitmap is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with exitmap. If not, see . 17 | 18 | """ 19 | Provides functions to keep track of scanning statistics. 20 | """ 21 | 22 | import logging 23 | from datetime import datetime 24 | 25 | from stem import CircStatus 26 | 27 | log = logging.getLogger(__name__) 28 | 29 | 30 | class Statistics(object): 31 | 32 | """ 33 | Keep track of scanning statistics. 34 | """ 35 | 36 | def __init__(self): 37 | """ 38 | Initialise a Statistics object. 39 | """ 40 | 41 | self.start_time = datetime.now() 42 | self.total_circuits = 0 43 | self.failed_circuits = 0 44 | self.successful_circuits = 0 45 | self.modules_run = 0 46 | self.finished_streams = 0 47 | self.failed_streams = 0 48 | 49 | def update_circs(self, circ_event): 50 | """ 51 | Update statistics with the given circuit event." 52 | """ 53 | 54 | if circ_event.status in [CircStatus.FAILED]: 55 | 56 | log.debug("Circuit failed because: %s" % str(circ_event.reason)) 57 | self.failed_circuits += 1 58 | 59 | elif circ_event.status in [CircStatus.BUILT]: 60 | 61 | self.successful_circuits += 1 62 | 63 | def print_progress(self, sampling=50): 64 | """ 65 | Print statistics about ongoing probing process. 66 | """ 67 | 68 | if (sampling == 0) or (self.finished_streams % sampling): 69 | return 70 | 71 | if self.total_circuits == 0: 72 | return 73 | 74 | percent_done = (self.successful_circuits / 75 | float(self.total_circuits)) * 100 76 | 77 | log.info("Probed %d out of %d exit relays, so we are %.2f%% done." % 78 | (self.successful_circuits, self.total_circuits, percent_done)) 79 | 80 | def __str__(self): 81 | """ 82 | Print the gathered statistics. 83 | """ 84 | 85 | percent = 0 86 | if self.total_circuits > 0: 87 | percent = (self.failed_circuits / float(self.total_circuits)) * 100 88 | 89 | return ("Ran %d module(s) in %s and %d/%d circuits failed (%.2f%%)." % 90 | (self.modules_run, 91 | str(datetime.now() - self.start_time), 92 | self.failed_circuits, 93 | self.total_circuits, 94 | percent)) 95 | -------------------------------------------------------------------------------- /src/torsocks.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015, 2016 Philipp Winter 2 | # 3 | # This file is part of exitmap. 4 | # 5 | # exitmap is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # exitmap is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with exitmap. If not, see . 17 | 18 | """ 19 | Provide a Tor-specific SOCKSv5 interface. 20 | """ 21 | 22 | import os 23 | import struct 24 | import socket 25 | import select 26 | import errno 27 | import logging 28 | import _socket 29 | import error 30 | import socks 31 | 32 | log = logging.getLogger(__name__) 33 | 34 | proxy_addr = None 35 | proxy_port = None 36 | queue = None 37 | circ_id = None 38 | 39 | _orig_getaddrinfo = socket.getaddrinfo 40 | orig_socket = socket.socket 41 | 42 | _ERRNO_RETRY = frozenset((errno.EAGAIN, errno.EWOULDBLOCK, 43 | errno.EINPROGRESS, errno.EINTR)) 44 | 45 | _LOCAL_SOCKETS = frozenset( 46 | getattr(socket, af) for af in [ 47 | 'AF_UNIX', 'AF_LOCAL', 48 | 'AF_ROUTE', 'AF_KEY', 'AF_ALG', 'AF_NETLINK' 49 | ] 50 | if hasattr(socket, af) 51 | ) 52 | 53 | # Fix PyPy 2.6.1 issue that Travis CI found. 54 | if not hasattr(errno, "ENOTSUP"): 55 | errno.ENOTSUP = 95 56 | 57 | # Map server-side SOCKSv5 errors to errno codes (as best we can; codes 58 | # 1 and 7 don't correspond to documented error codes for connect(2)) 59 | socks5_errors = { 60 | 0x00: 0, # Success 61 | 0x01: errno.EIO, # General failure 62 | 0x02: errno.EACCES, # Connection not allowed by ruleset 63 | 0x03: errno.ENETUNREACH, # Network unreachable 64 | 0x04: errno.EHOSTUNREACH, # Host unreachable 65 | 0x05: errno.ECONNREFUSED, # Connection refused by destination host 66 | 0x06: errno.ETIMEDOUT, # TTL expired 67 | 0x07: errno.ENOTSUP, # Command not supported / protocol error 68 | 0x08: errno.EAFNOSUPPORT, # Address type not supported 69 | } 70 | 71 | 72 | def send_queue(sock_name): 73 | """ 74 | Inform caller about our newly created socket. 75 | """ 76 | 77 | global queue, circ_id 78 | assert (queue is not None) and (circ_id is not None) 79 | queue.put([circ_id, sock_name]) 80 | 81 | class _Torsocket(socks.socksocket): 82 | def __init__(self, *args, **kwargs): 83 | super(_Torsocket, self).__init__(*args, **kwargs) 84 | orig_neg = self._proxy_negotiators[2] # This is the original function 85 | def ourneg(*args, **kwargs): 86 | "Our modified function to add data to the queue" 87 | try: 88 | # we are adding to the queue before as orig_neg will also do 89 | # the actual connection to the destination inside. 90 | # args[0] is the original socket to the proxy address 91 | send_queue(args[0].getsockname()) 92 | orig_neg(*args, **kwargs) 93 | except Exception as e: 94 | log.debug("Error in custom negotiation function: {}".format(e)) 95 | self._proxy_negotiators[2] = ourneg 96 | 97 | def negotiate(self): 98 | proxy_type, addr, port, rdns, username, password = self.proxy 99 | socks._BaseSocket.connect(self, (addr, port)) 100 | socks._BaseSocket.sendall(self, struct.pack('BBB', 0x05, 0x01, 0x00)) 101 | socks._BaseSocket.recv(self, 2) 102 | 103 | def resolve(self, hostname): 104 | "Resolves the given domain name over the proxy" 105 | host = hostname.encode("utf-8") 106 | # First connect to the local proxy 107 | self.negotiate() 108 | send_queue(socks._BaseSocket.getsockname(self)) 109 | req = struct.pack('BBB', 0x05, 0xF0, 0x00) 110 | req += chr(0x03).encode() + chr(len(host)).encode() + host 111 | req = req + struct.pack(">H", 8444) 112 | socks._BaseSocket.sendall(self, req) 113 | # Get the response 114 | ip = "" 115 | resp = socks._BaseSocket.recv(self, 4) 116 | if resp[0:1] != chr(0x05).encode(): 117 | socks._BaseSocket.close(self) 118 | raise error.SOCKSv5Error("SOCKS Server error") 119 | elif resp[1:2] != chr(0x00).encode(): 120 | # Connection failed 121 | socks._BaseSocket.close(self) 122 | if ord(resp[1:2])<=8: 123 | raise error.SOCKSv5Error("SOCKS Server error {}".format(ord(resp[1:2]))) 124 | else: 125 | raise error.SOCKSv5Error("SOCKS Server error 9") 126 | # Get the bound address/port 127 | elif resp[3:4] == chr(0x01).encode(): 128 | ip = socket.inet_ntoa(socks._BaseSocket.recv(self, 4)) 129 | elif resp[3:4] == chr(0x03).encode(): 130 | resp = resp + socks._BaseSocket.recv(self, 1) 131 | ip = socks._BaseSocket.recv(self, ord(resp[4:5])) 132 | else: 133 | socks._BaseSocket.close(self) 134 | raise error.SOCKSv5Error("SOCKS Server error.") 135 | boundport = struct.unpack(">H", socks._BaseSocket.recv(self, 2))[0] 136 | socks._BaseSocket.close(self) 137 | return ip 138 | 139 | 140 | 141 | def torsocket(family=socket.AF_INET, type=socket.SOCK_STREAM, 142 | proto=0, _sock=None): 143 | """ 144 | Factory function usable as a monkey-patch for socket.socket. 145 | """ 146 | 147 | # Pass through local sockets. 148 | if family in _LOCAL_SOCKETS: 149 | return orig_socket(family, type, proto, _sock) 150 | 151 | # Tor only supports AF_INET sockets. 152 | if family != socket.AF_INET: 153 | raise socket.error(errno.EAFNOSUPPORT, os.strerror(errno.EAFNOSUPPORT)) 154 | 155 | # Tor only supports SOCK_STREAM sockets. 156 | if type != socket.SOCK_STREAM: 157 | raise socket.error(errno.ESOCKTNOSUPPORT, 158 | os.strerror(errno.ESOCKTNOSUPPORT)) 159 | 160 | # Acceptable values for PROTO are 0 and IPPROTO_TCP. 161 | if proto not in (0, socket.IPPROTO_TCP): 162 | raise socket.error(errno.EPROTONOSUPPORT, 163 | os.strerror(errno.EPROTONOSUPPORT)) 164 | 165 | return _Torsocket(family, type, proto, _sock) 166 | 167 | def getaddrinfo(*args): 168 | return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] 169 | 170 | class MonkeyPatchedSocket(object): 171 | """ 172 | Context manager which monkey-patches socket.socket with 173 | the above torsocket(). It also sets up this module's 174 | global state. 175 | """ 176 | def __init__(self, queue, circ_id, socks_port, socks_addr="127.0.0.1"): 177 | self._queue = queue 178 | self._circ_id = circ_id 179 | self._socks_addr = socks_addr 180 | self._socks_port = socks_port 181 | 182 | self._orig_queue = None 183 | self._orig_circ_id = None 184 | self._orig_proxy_addr = None 185 | self._orig_proxy_port = None 186 | self._orig_socket = None 187 | 188 | def __enter__(self): 189 | global queue, circ_id, proxy_addr, proxy_port, socket, torsocket 190 | 191 | # Make sure __exit__ can put everything back just as it was. 192 | self._orig_queue = queue 193 | self._orig_circ_id = circ_id 194 | self._orig_proxy_addr = proxy_addr 195 | self._orig_proxy_port = proxy_port 196 | self._orig_socket = socket.socket 197 | 198 | queue = self._queue 199 | circ_id = self._circ_id 200 | proxy_addr = self._socks_addr 201 | proxy_port = self._socks_port 202 | socks.set_default_proxy(socks.SOCKS5, proxy_addr, proxy_port, True, None, None) 203 | socket.socket = torsocket 204 | socket.getaddrinfo = getaddrinfo 205 | 206 | return self 207 | 208 | def __exit__(self, *dontcare): 209 | global queue, circ_id, proxy_addr, proxy_port, socket 210 | 211 | queue = self._orig_queue 212 | circ_id = self._orig_circ_id 213 | proxy_addr = self._orig_proxy_addr 214 | proxy_port = self._orig_proxy_port 215 | socket.socket = self._orig_socket 216 | socket.getaddrinfo = _orig_getaddrinfo 217 | 218 | return False 219 | -------------------------------------------------------------------------------- /src/util.py: -------------------------------------------------------------------------------- 1 | # Copyright 2013-2017 Philipp Winter 2 | # 3 | # This file is part of exitmap. 4 | # 5 | # exitmap is free software: you can redistribute it and/or modify 6 | # it under the terms of the GNU General Public License as published by 7 | # the Free Software Foundation, either version 3 of the License, or 8 | # (at your option) any later version. 9 | # 10 | # exitmap is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | # GNU General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU General Public License 16 | # along with exitmap. If not, see . 17 | 18 | """ 19 | Provides utility functions. 20 | """ 21 | 22 | import os 23 | import re 24 | import logging 25 | try: 26 | import urllib2 27 | except ImportError: 28 | import urllib.request as urllib2 29 | import json 30 | import tempfile 31 | import errno 32 | 33 | from stem.descriptor.reader import DescriptorReader 34 | 35 | 36 | log = logging.getLogger(__name__) 37 | 38 | # Holds the directory to which we can write temporary analysis results. 39 | 40 | analysis_dir = None 41 | 42 | 43 | def parse_log_lines(ports, log_line): 44 | """ 45 | Extract the SOCKS and control port from Tor's log output. 46 | 47 | Both ports are written to the given dictionary. 48 | """ 49 | 50 | log.debug("Tor says: %s" % log_line) 51 | 52 | if re.search(r"^.*Bootstrapped \d+%.*$", log_line): 53 | log.info(re.sub(r"^.*(Bootstrapped \d+%.*)$", r"Tor \1", log_line)) 54 | 55 | socks_pattern = "Socks listener listening on port ([0-9]{1,5})." 56 | control_pattern = "Control listener listening on port ([0-9]{1,5})." 57 | 58 | match = re.search(socks_pattern, log_line) 59 | if match: 60 | ports["socks"] = int(match.group(1)) 61 | log.debug("Tor uses port %d as SOCKS port." % ports["socks"]) 62 | 63 | match = re.search(control_pattern, log_line) 64 | if match: 65 | ports["control"] = int(match.group(1)) 66 | log.debug("Tor uses port %d as control port." % ports["control"]) 67 | 68 | 69 | def relay_in_consensus(fingerprint, cached_consensus_path): 70 | """ 71 | Check if a relay is part of the consensus. 72 | 73 | If the relay identified by `fingerprint' is part of the given `consensus', 74 | True is returned. If not, False is returned. 75 | """ 76 | 77 | fingerprint = fingerprint.upper() 78 | 79 | with DescriptorReader(cached_consensus_path) as reader: 80 | for descriptor in reader: 81 | if descriptor.fingerprint == fingerprint: 82 | return True 83 | 84 | return False 85 | 86 | 87 | def get_source_port(stream_line): 88 | """ 89 | Extract the source port from a stream event. 90 | """ 91 | 92 | pattern = "SOURCE_ADDR=[0-9\.]{7,15}:([0-9]{1,5})" 93 | match = re.search(pattern, stream_line) 94 | 95 | if match: 96 | return int(match.group(1)) 97 | 98 | return None 99 | 100 | 101 | def extract_pattern(line, pattern): 102 | """ 103 | Look for the given 'pattern' in 'line'. 104 | 105 | If it is found, the match is returned. Otherwise, 'None' is returned. 106 | """ 107 | 108 | match = re.search(pattern, line) 109 | 110 | if match: 111 | return match.group(1) 112 | 113 | return None 114 | 115 | 116 | def get_relays_in_country(country_code): 117 | """ 118 | Return a list of the fingerprints of all relays in the given country code. 119 | 120 | The fingerprints are obtained by querying Onionoo. 121 | """ 122 | 123 | country_code = country_code.lower() 124 | onionoo_url = "https://onionoo.torproject.org/details?country=" 125 | 126 | log.info("Attempting to fetch all relays with country code \"%s\" " 127 | "from Onionoo." % country_code) 128 | 129 | f = urllib2.urlopen("%s%s" % (onionoo_url, country_code)) 130 | data = f.read().decode('utf-8') 131 | response = json.loads(data) 132 | 133 | fingerprints = [desc["fingerprint"] for desc in response["relays"]] 134 | 135 | log.info("Onionoo gave us %d (exit and non-exit) fingerprints." % 136 | len(fingerprints)) 137 | 138 | return fingerprints 139 | 140 | 141 | def exiturl(exit_fpr): 142 | """ 143 | Return a Metrics link for the exit relay fingerprint. 144 | """ 145 | 146 | return "" % exit_fpr 147 | 148 | 149 | def dump_to_file(blurb, exit_fpr): 150 | """ 151 | Dump the given blurb to a randomly generated file which contains exit_fpr. 152 | 153 | This function is useful to save data obtained from bad exit relays to file 154 | for later analysis. 155 | """ 156 | if analysis_dir is None: 157 | fd, file_name = tempfile.mkstemp(prefix="%s_" % exit_fpr) 158 | 159 | else: 160 | try: 161 | os.makedirs(analysis_dir) 162 | except OSError as err: 163 | if err.errno != errno.EEXIST: 164 | raise 165 | fd, file_name = tempfile.mkstemp(prefix="%s_" % exit_fpr, 166 | dir=analysis_dir) 167 | 168 | try: 169 | with open(file_name, "w") as fd: 170 | fd.write(blurb) 171 | except IOError as err: 172 | log.warning("Couldn't write to \"%s\": %s" % (file_name, err)) 173 | return None 174 | 175 | log.debug("Wrote %d-length blurb to file \"%s\"." % 176 | (len(blurb), file_name)) 177 | 178 | return file_name 179 | 180 | 181 | def new_request(url, data=None): 182 | """ 183 | Return a request object whose HTTP header resembles TorBrowser. 184 | """ 185 | 186 | request = urllib2.Request(url, data) 187 | 188 | # Try to resemble the HTTP request of TorBrowser as closely as possible. 189 | # Note that the order of header fields is also relevant but urllib2 uses a 190 | # dictionary for headers, which is orderless. 191 | 192 | request.add_header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:60.0) " 193 | "Gecko/20100101 Firefox/60.0") 194 | request.add_header("Accept", "text/html,application/xhtml+xml," 195 | "application/xml;q=0.9,*/*;q=0.8") 196 | request.add_header("Accept-Language", "en-US,en;q=0.5") 197 | request.add_header("Accept-Encoding", "gzip, deflate, br") 198 | request.add_header("Upgrade-Insecure-Requests", "1") 199 | 200 | return request 201 | -------------------------------------------------------------------------------- /test/run_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | """ 4 | Runs our test suite. Presently this is just static checks (pyflakes and pep8), 5 | but in the future might be extended to run unit tests. 6 | """ 7 | 8 | import os 9 | 10 | import stem.util.conf 11 | import stem.util.test_tools 12 | 13 | EXITMAP_BASE = os.path.dirname(__file__) 14 | 15 | 16 | def main(): 17 | test_config = stem.util.conf.get_config("test") 18 | test_config.load(os.path.join(EXITMAP_BASE, "test_settings.cfg")) 19 | 20 | orphaned_pyc = stem.util.test_tools.clean_orphaned_pyc(EXITMAP_BASE) 21 | 22 | for path in orphaned_pyc: 23 | print "Deleted orphaned pyc file: %s" % path 24 | 25 | # TODO: Uncomment to run unit tests in ./tests/* 26 | # 27 | # tests = unittest.defaultTestLoader.discover('test', pattern='*.py') 28 | # test_runner = unittest.TextTestRunner() 29 | # test_runner.run(tests) 30 | 31 | print 32 | 33 | static_check_issues = {} 34 | 35 | if stem.util.test_tools.is_pyflakes_available(): 36 | for path, issues in stem.util.test_tools.get_pyflakes_issues([EXITMAP_BASE]).items(): 37 | for issue in issues: 38 | static_check_issues.setdefault(path, []).append(issue) 39 | else: 40 | print "Pyflakes unavailable. Please install with 'sudo pip install pyflakes'." 41 | 42 | if stem.util.test_tools.is_pep8_available(): 43 | for path, issues in stem.util.test_tools.get_stylistic_issues([EXITMAP_BASE]).items(): 44 | for issue in issues: 45 | static_check_issues.setdefault(path, []).append(issue) 46 | else: 47 | print "Pep8 unavailable. Please install with 'sudo pip install pep8'." 48 | 49 | if static_check_issues: 50 | print "STATIC CHECKS" 51 | print 52 | 53 | for file_path in static_check_issues: 54 | print "* %s" % file_path 55 | 56 | for line_number, msg, code in static_check_issues[file_path]: 57 | line_count = "%-4s" % line_number 58 | print " line %s - %s" % (line_count, msg) 59 | 60 | print 61 | 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /test/test_relayselector.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | # Copyright 2016 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | """ Unit tests for the relay selector module.""" 20 | 21 | import unittest 22 | import sys 23 | sys.path.insert(0, 'src/') 24 | import relayselector 25 | 26 | 27 | class TestRelaySelector(unittest.TestCase): 28 | """Test the torsocks module.""" 29 | 30 | def test_get_exits(self): 31 | with self.assertRaises(SystemExit) as exits: 32 | relayselector.get_exits('/tmp', 33 | good_exit=True, 34 | bad_exit=True, 35 | version=None, 36 | nickname=None, 37 | address=None, 38 | country_code='at', 39 | requested_exits=None, 40 | destinations=None) 41 | self.assertEqual(exits.exception.code, 1) 42 | 43 | 44 | if __name__ == '__main__': 45 | unittest.main() 46 | -------------------------------------------------------------------------------- /test/test_settings.cfg: -------------------------------------------------------------------------------- 1 | # PEP8 compliance issues that we're ignoring. 2 | pep8.ignore E402 3 | pep8.ignore E303 4 | pep8.ignore E501 5 | -------------------------------------------------------------------------------- /test/test_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | # Copyright 2015-2016 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | """ 20 | Implements unit tests. 21 | """ 22 | 23 | import unittest 24 | import stem.control 25 | from stem import CircStatus 26 | import sys 27 | sys.path.insert(0, 'src/') 28 | import stats 29 | 30 | 31 | class TestStats(unittest.TestCase): 32 | """Test the stats module.""" 33 | 34 | def setUp(self): 35 | self.stats = stats.Statistics() 36 | 37 | def test_stats(self): 38 | self.stats.print_progress(sampling=0) 39 | self.stats.print_progress 40 | self.assertTrue(str(self.stats)) 41 | 42 | circ_event = stem.response.events.CircuitEvent("foo", "bar") 43 | circ_event.status = CircStatus.FAILED 44 | circ_event.reason = "foo" 45 | 46 | self.stats.update_circs(circ_event) 47 | self.assertEqual(self.stats.failed_circuits, 1) 48 | 49 | circ_event.status = CircStatus.BUILT 50 | 51 | self.stats.update_circs(circ_event) 52 | self.assertEqual(self.stats.successful_circuits, 1) 53 | 54 | 55 | if __name__ == '__main__': 56 | unittest.main() 57 | -------------------------------------------------------------------------------- /test/test_torsocks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | # Copyright 2016 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | """ Unit tests for the torsocks module.""" 20 | 21 | import unittest 22 | import sys 23 | sys.path.insert(0, 'src/') 24 | import torsocks 25 | from error import SOCKSv5Error 26 | 27 | 28 | class TestTorsocks(unittest.TestCase): 29 | """Test the torsocks module.""" 30 | 31 | def test_send_queue(self): 32 | self.assertRaises(AssertionError, torsocks.send_queue, 33 | ('127.0.0.1', 38662)) 34 | 35 | 36 | if __name__ == '__main__': 37 | unittest.main() 38 | -------------------------------------------------------------------------------- /test/test_util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python2 2 | 3 | # Copyright 2015-2016 Philipp Winter 4 | # 5 | # This file is part of exitmap. 6 | # 7 | # exitmap is free software: you can redistribute it and/or modify 8 | # it under the terms of the GNU General Public License as published by 9 | # the Free Software Foundation, either version 3 of the License, or 10 | # (at your option) any later version. 11 | # 12 | # exitmap is distributed in the hope that it will be useful, 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | # GNU General Public License for more details. 16 | # 17 | # You should have received a copy of the GNU General Public License 18 | # along with exitmap. If not, see . 19 | """ 20 | Implements unit tests. 21 | """ 22 | 23 | import unittest 24 | import sys 25 | sys.path.insert(0, 'src/') 26 | import util 27 | 28 | 29 | class TestUtil(unittest.TestCase): 30 | """Test the util module.""" 31 | 32 | def test_get_relays_in_country(self): 33 | try: 34 | relays1 = util.get_relays_in_country("at") 35 | except Exception as err: 36 | return 37 | self.assertTrue(len(relays1) > 0) 38 | 39 | try: 40 | relays2 = util.get_relays_in_country("AT") 41 | except Exception as err: 42 | return 43 | self.assertTrue(len(relays1) == len(relays2)) 44 | 45 | try: 46 | bogus = util.get_relays_in_country("foo") 47 | except Exception as err: 48 | return 49 | self.assertEqual(bogus, []) 50 | 51 | def test_get_source_port(self): 52 | self.assertEqual(util.get_source_port("SOURCE_ADDR=" 53 | "255.255.255.255:0"), 0) 54 | self.assertEqual(util.get_source_port("SOURCE_ADDR=1.1.1.1:1"), 1) 55 | self.assertEqual(util.get_source_port("SOURCE_ADDR=1.1.1.1:" 56 | "65535"), 65535) 57 | self.assertIsNone(util.get_source_port("")) 58 | 59 | def test_exiturl(self): 60 | self.assertEqual(util.exiturl("foo"), ("")) 62 | self.assertEqual(util.exiturl(4), ("")) 64 | 65 | def test_extract_pattern(self): 66 | extract_pattern1 = util.extract_pattern("Connection on fd 4 originat" 67 | "ing from 444:0000", "Connec" 68 | "tion on fd [0-9]+ originati" 69 | "ng from [^:]+:([0-9]{1,5})") 70 | self.assertEqual(extract_pattern1, "0000") 71 | self.assertIsNone(util.extract_pattern("", "")) 73 | 74 | 75 | def test_new_request(self): 76 | result = util.new_request("https://atlas.torproject.org", "test") 77 | self.assertEqual("https://atlas.torproject.org", result.get_full_url()) 78 | self.assertTrue(result.has_header("User-agent")) 79 | self.assertTrue(result.has_header("Accept")) 80 | self.assertTrue(result.has_header("Accept-language")) 81 | self.assertTrue(result.has_header("Accept-encoding")) 82 | 83 | def test_parse_log_lines(self): 84 | ports = {"socks": -1, "control": -1} 85 | util.parse_log_lines(ports, "foo Bootstrapped 444%foo tor" 86 | "Socks listener listening on port 8000.") 87 | util.parse_log_lines(ports, "Control listener listening on port 9000.") 88 | self.assertEqual(ports["socks"], 8000) 89 | self.assertEqual(ports["control"], 9000) 90 | 91 | 92 | if __name__ == '__main__': 93 | unittest.main() 94 | --------------------------------------------------------------------------------