├── COPYING ├── LICENCE.txt ├── README.md ├── VERSION ├── examples ├── analyse_text.py └── stepfun.json ├── future-work.md ├── implementation-notes.md ├── setup.py ├── src └── pysfn │ ├── __init__.py │ ├── definition.py │ └── tools │ ├── __init__.py │ ├── compile.py │ └── gen_lambda.py └── tests ├── __init__.py ├── test_analyse_text.py └── test_pysfnc.py /COPYING: -------------------------------------------------------------------------------- 1 | Code 2 | 3 | examples/analyse_text.py 4 | all in src/ 5 | all in tests/ 6 | setup.py 7 | 8 | is hereby made available under the terms of the GNU General Public 9 | License as published by the Free Software Foundation, either version 3 10 | of the License, or (at your option) any later version. 11 | 12 | The file 13 | 14 | LICENCE.txt 15 | 16 | contains a copy of the GNU General Public License v3; its copyright is 17 | held by the Free Software Foundation; it is is freely distributable in 18 | unmodified form. 19 | 20 | 21 | ------------------------------------------------------------------------ 22 | 23 | The non-code content 24 | 25 | README.md 26 | future-work.md 27 | implementation-notes.md 28 | 29 | is hereby made available under the Creative Commons Attribution 30 | Share-Alike licence. A summary of this licence, and link to the full 31 | version, can be found here: 32 | 33 | http://creativecommons.org/licenses/by-sa/4.0/ 34 | 35 | 36 | ------------------------------------------------------------------------ 37 | 38 | The generated content 39 | 40 | examples/stepfun.json 41 | 42 | is hereby placed into the public domain. 43 | -------------------------------------------------------------------------------- /LICENCE.txt: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU General Public License is a free, copyleft license for 11 | software and other kinds of works. 12 | 13 | The licenses for most software and other practical works are designed 14 | to take away your freedom to share and change the works. By contrast, 15 | the GNU General Public License is intended to guarantee your freedom to 16 | share and change all versions of a program--to make sure it remains free 17 | software for all its users. We, the Free Software Foundation, use the 18 | GNU General Public License for most of our software; it applies also to 19 | any other work released this way by its authors. You can apply it to 20 | your programs, too. 21 | 22 | When we speak of free software, we are referring to freedom, not 23 | price. Our General Public Licenses are designed to make sure that you 24 | have the freedom to distribute copies of free software (and charge for 25 | them if you wish), that you receive source code or can get it if you 26 | want it, that you can change the software or use pieces of it in new 27 | free programs, and that you know you can do these things. 28 | 29 | To protect your rights, we need to prevent others from denying you 30 | these rights or asking you to surrender the rights. Therefore, you have 31 | certain responsibilities if you distribute copies of the software, or if 32 | you modify it: responsibilities to respect the freedom of others. 33 | 34 | For example, if you distribute copies of such a program, whether 35 | gratis or for a fee, you must pass on to the recipients the same 36 | freedoms that you received. You must make sure that they, too, receive 37 | or can get the source code. And you must show them these terms so they 38 | know their rights. 39 | 40 | Developers that use the GNU GPL protect your rights with two steps: 41 | (1) assert copyright on the software, and (2) offer you this License 42 | giving you legal permission to copy, distribute and/or modify it. 43 | 44 | For the developers' and authors' protection, the GPL clearly explains 45 | that there is no warranty for this free software. For both users' and 46 | authors' sake, the GPL requires that modified versions be marked as 47 | changed, so that their problems will not be attributed erroneously to 48 | authors of previous versions. 49 | 50 | Some devices are designed to deny users access to install or run 51 | modified versions of the software inside them, although the manufacturer 52 | can do so. This is fundamentally incompatible with the aim of 53 | protecting users' freedom to change the software. The systematic 54 | pattern of such abuse occurs in the area of products for individuals to 55 | use, which is precisely where it is most unacceptable. Therefore, we 56 | have designed this version of the GPL to prohibit the practice for those 57 | products. If such problems arise substantially in other domains, we 58 | stand ready to extend this provision to those domains in future versions 59 | of the GPL, as needed to protect the freedom of users. 60 | 61 | Finally, every program is threatened constantly by software patents. 62 | States should not allow patents to restrict development and use of 63 | software on general-purpose computers, but in those that do, we wish to 64 | avoid the special danger that patents applied to a free program could 65 | make it effectively proprietary. To prevent this, the GPL assures that 66 | patents cannot be used to render the program non-free. 67 | 68 | The precise terms and conditions for copying, distribution and 69 | modification follow. 70 | 71 | TERMS AND CONDITIONS 72 | 73 | 0. Definitions. 74 | 75 | "This License" refers to version 3 of the GNU General Public License. 76 | 77 | "Copyright" also means copyright-like laws that apply to other kinds of 78 | works, such as semiconductor masks. 79 | 80 | "The Program" refers to any copyrightable work licensed under this 81 | License. Each licensee is addressed as "you". "Licensees" and 82 | "recipients" may be individuals or organizations. 83 | 84 | To "modify" a work means to copy from or adapt all or part of the work 85 | in a fashion requiring copyright permission, other than the making of an 86 | exact copy. The resulting work is called a "modified version" of the 87 | earlier work or a work "based on" the earlier work. 88 | 89 | A "covered work" means either the unmodified Program or a work based 90 | on the Program. 91 | 92 | To "propagate" a work means to do anything with it that, without 93 | permission, would make you directly or secondarily liable for 94 | infringement under applicable copyright law, except executing it on a 95 | computer or modifying a private copy. Propagation includes copying, 96 | distribution (with or without modification), making available to the 97 | public, and in some countries other activities as well. 98 | 99 | To "convey" a work means any kind of propagation that enables other 100 | parties to make or receive copies. Mere interaction with a user through 101 | a computer network, with no transfer of a copy, is not conveying. 102 | 103 | An interactive user interface displays "Appropriate Legal Notices" 104 | to the extent that it includes a convenient and prominently visible 105 | feature that (1) displays an appropriate copyright notice, and (2) 106 | tells the user that there is no warranty for the work (except to the 107 | extent that warranties are provided), that licensees may convey the 108 | work under this License, and how to view a copy of this License. If 109 | the interface presents a list of user commands or options, such as a 110 | menu, a prominent item in the list meets this criterion. 111 | 112 | 1. Source Code. 113 | 114 | The "source code" for a work means the preferred form of the work 115 | for making modifications to it. "Object code" means any non-source 116 | form of a work. 117 | 118 | A "Standard Interface" means an interface that either is an official 119 | standard defined by a recognized standards body, or, in the case of 120 | interfaces specified for a particular programming language, one that 121 | is widely used among developers working in that language. 122 | 123 | The "System Libraries" of an executable work include anything, other 124 | than the work as a whole, that (a) is included in the normal form of 125 | packaging a Major Component, but which is not part of that Major 126 | Component, and (b) serves only to enable use of the work with that 127 | Major Component, or to implement a Standard Interface for which an 128 | implementation is available to the public in source code form. A 129 | "Major Component", in this context, means a major essential component 130 | (kernel, window system, and so on) of the specific operating system 131 | (if any) on which the executable work runs, or a compiler used to 132 | produce the work, or an object code interpreter used to run it. 133 | 134 | The "Corresponding Source" for a work in object code form means all 135 | the source code needed to generate, install, and (for an executable 136 | work) run the object code and to modify the work, including scripts to 137 | control those activities. However, it does not include the work's 138 | System Libraries, or general-purpose tools or generally available free 139 | programs which are used unmodified in performing those activities but 140 | which are not part of the work. For example, Corresponding Source 141 | includes interface definition files associated with source files for 142 | the work, and the source code for shared libraries and dynamically 143 | linked subprograms that the work is specifically designed to require, 144 | such as by intimate data communication or control flow between those 145 | subprograms and other parts of the work. 146 | 147 | The Corresponding Source need not include anything that users 148 | can regenerate automatically from other parts of the Corresponding 149 | Source. 150 | 151 | The Corresponding Source for a work in source code form is that 152 | same work. 153 | 154 | 2. Basic Permissions. 155 | 156 | All rights granted under this License are granted for the term of 157 | copyright on the Program, and are irrevocable provided the stated 158 | conditions are met. This License explicitly affirms your unlimited 159 | permission to run the unmodified Program. The output from running a 160 | covered work is covered by this License only if the output, given its 161 | content, constitutes a covered work. This License acknowledges your 162 | rights of fair use or other equivalent, as provided by copyright law. 163 | 164 | You may make, run and propagate covered works that you do not 165 | convey, without conditions so long as your license otherwise remains 166 | in force. You may convey covered works to others for the sole purpose 167 | of having them make modifications exclusively for you, or provide you 168 | with facilities for running those works, provided that you comply with 169 | the terms of this License in conveying all material for which you do 170 | not control copyright. Those thus making or running the covered works 171 | for you must do so exclusively on your behalf, under your direction 172 | and control, on terms that prohibit them from making any copies of 173 | your copyrighted material outside their relationship with you. 174 | 175 | Conveying under any other circumstances is permitted solely under 176 | the conditions stated below. Sublicensing is not allowed; section 10 177 | makes it unnecessary. 178 | 179 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 180 | 181 | No covered work shall be deemed part of an effective technological 182 | measure under any applicable law fulfilling obligations under article 183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 184 | similar laws prohibiting or restricting circumvention of such 185 | measures. 186 | 187 | When you convey a covered work, you waive any legal power to forbid 188 | circumvention of technological measures to the extent such circumvention 189 | is effected by exercising rights under this License with respect to 190 | the covered work, and you disclaim any intention to limit operation or 191 | modification of the work as a means of enforcing, against the work's 192 | users, your or third parties' legal rights to forbid circumvention of 193 | technological measures. 194 | 195 | 4. Conveying Verbatim Copies. 196 | 197 | You may convey verbatim copies of the Program's source code as you 198 | receive it, in any medium, provided that you conspicuously and 199 | appropriately publish on each copy an appropriate copyright notice; 200 | keep intact all notices stating that this License and any 201 | non-permissive terms added in accord with section 7 apply to the code; 202 | keep intact all notices of the absence of any warranty; and give all 203 | recipients a copy of this License along with the Program. 204 | 205 | You may charge any price or no price for each copy that you convey, 206 | and you may offer support or warranty protection for a fee. 207 | 208 | 5. Conveying Modified Source Versions. 209 | 210 | You may convey a work based on the Program, or the modifications to 211 | produce it from the Program, in the form of source code under the 212 | terms of section 4, provided that you also meet all of these conditions: 213 | 214 | a) The work must carry prominent notices stating that you modified 215 | it, and giving a relevant date. 216 | 217 | b) The work must carry prominent notices stating that it is 218 | released under this License and any conditions added under section 219 | 7. This requirement modifies the requirement in section 4 to 220 | "keep intact all notices". 221 | 222 | c) You must license the entire work, as a whole, under this 223 | License to anyone who comes into possession of a copy. This 224 | License will therefore apply, along with any applicable section 7 225 | additional terms, to the whole of the work, and all its parts, 226 | regardless of how they are packaged. This License gives no 227 | permission to license the work in any other way, but it does not 228 | invalidate such permission if you have separately received it. 229 | 230 | d) If the work has interactive user interfaces, each must display 231 | Appropriate Legal Notices; however, if the Program has interactive 232 | interfaces that do not display Appropriate Legal Notices, your 233 | work need not make them do so. 234 | 235 | A compilation of a covered work with other separate and independent 236 | works, which are not by their nature extensions of the covered work, 237 | and which are not combined with it such as to form a larger program, 238 | in or on a volume of a storage or distribution medium, is called an 239 | "aggregate" if the compilation and its resulting copyright are not 240 | used to limit the access or legal rights of the compilation's users 241 | beyond what the individual works permit. Inclusion of a covered work 242 | in an aggregate does not cause this License to apply to the other 243 | parts of the aggregate. 244 | 245 | 6. Conveying Non-Source Forms. 246 | 247 | You may convey a covered work in object code form under the terms 248 | of sections 4 and 5, provided that you also convey the 249 | machine-readable Corresponding Source under the terms of this License, 250 | in one of these ways: 251 | 252 | a) Convey the object code in, or embodied in, a physical product 253 | (including a physical distribution medium), accompanied by the 254 | Corresponding Source fixed on a durable physical medium 255 | customarily used for software interchange. 256 | 257 | b) Convey the object code in, or embodied in, a physical product 258 | (including a physical distribution medium), accompanied by a 259 | written offer, valid for at least three years and valid for as 260 | long as you offer spare parts or customer support for that product 261 | model, to give anyone who possesses the object code either (1) a 262 | copy of the Corresponding Source for all the software in the 263 | product that is covered by this License, on a durable physical 264 | medium customarily used for software interchange, for a price no 265 | more than your reasonable cost of physically performing this 266 | conveying of source, or (2) access to copy the 267 | Corresponding Source from a network server at no charge. 268 | 269 | c) Convey individual copies of the object code with a copy of the 270 | written offer to provide the Corresponding Source. This 271 | alternative is allowed only occasionally and noncommercially, and 272 | only if you received the object code with such an offer, in accord 273 | with subsection 6b. 274 | 275 | d) Convey the object code by offering access from a designated 276 | place (gratis or for a charge), and offer equivalent access to the 277 | Corresponding Source in the same way through the same place at no 278 | further charge. You need not require recipients to copy the 279 | Corresponding Source along with the object code. If the place to 280 | copy the object code is a network server, the Corresponding Source 281 | may be on a different server (operated by you or a third party) 282 | that supports equivalent copying facilities, provided you maintain 283 | clear directions next to the object code saying where to find the 284 | Corresponding Source. Regardless of what server hosts the 285 | Corresponding Source, you remain obligated to ensure that it is 286 | available for as long as needed to satisfy these requirements. 287 | 288 | e) Convey the object code using peer-to-peer transmission, provided 289 | you inform other peers where the object code and Corresponding 290 | Source of the work are being offered to the general public at no 291 | charge under subsection 6d. 292 | 293 | A separable portion of the object code, whose source code is excluded 294 | from the Corresponding Source as a System Library, need not be 295 | included in conveying the object code work. 296 | 297 | A "User Product" is either (1) a "consumer product", which means any 298 | tangible personal property which is normally used for personal, family, 299 | or household purposes, or (2) anything designed or sold for incorporation 300 | into a dwelling. In determining whether a product is a consumer product, 301 | doubtful cases shall be resolved in favor of coverage. For a particular 302 | product received by a particular user, "normally used" refers to a 303 | typical or common use of that class of product, regardless of the status 304 | of the particular user or of the way in which the particular user 305 | actually uses, or expects or is expected to use, the product. A product 306 | is a consumer product regardless of whether the product has substantial 307 | commercial, industrial or non-consumer uses, unless such uses represent 308 | the only significant mode of use of the product. 309 | 310 | "Installation Information" for a User Product means any methods, 311 | procedures, authorization keys, or other information required to install 312 | and execute modified versions of a covered work in that User Product from 313 | a modified version of its Corresponding Source. The information must 314 | suffice to ensure that the continued functioning of the modified object 315 | code is in no case prevented or interfered with solely because 316 | modification has been made. 317 | 318 | If you convey an object code work under this section in, or with, or 319 | specifically for use in, a User Product, and the conveying occurs as 320 | part of a transaction in which the right of possession and use of the 321 | User Product is transferred to the recipient in perpetuity or for a 322 | fixed term (regardless of how the transaction is characterized), the 323 | Corresponding Source conveyed under this section must be accompanied 324 | by the Installation Information. But this requirement does not apply 325 | if neither you nor any third party retains the ability to install 326 | modified object code on the User Product (for example, the work has 327 | been installed in ROM). 328 | 329 | The requirement to provide Installation Information does not include a 330 | requirement to continue to provide support service, warranty, or updates 331 | for a work that has been modified or installed by the recipient, or for 332 | the User Product in which it has been modified or installed. Access to a 333 | network may be denied when the modification itself materially and 334 | adversely affects the operation of the network or violates the rules and 335 | protocols for communication across the network. 336 | 337 | Corresponding Source conveyed, and Installation Information provided, 338 | in accord with this section must be in a format that is publicly 339 | documented (and with an implementation available to the public in 340 | source code form), and must require no special password or key for 341 | unpacking, reading or copying. 342 | 343 | 7. Additional Terms. 344 | 345 | "Additional permissions" are terms that supplement the terms of this 346 | License by making exceptions from one or more of its conditions. 347 | Additional permissions that are applicable to the entire Program shall 348 | be treated as though they were included in this License, to the extent 349 | that they are valid under applicable law. If additional permissions 350 | apply only to part of the Program, that part may be used separately 351 | under those permissions, but the entire Program remains governed by 352 | this License without regard to the additional permissions. 353 | 354 | When you convey a copy of a covered work, you may at your option 355 | remove any additional permissions from that copy, or from any part of 356 | it. (Additional permissions may be written to require their own 357 | removal in certain cases when you modify the work.) You may place 358 | additional permissions on material, added by you to a covered work, 359 | for which you have or can give appropriate copyright permission. 360 | 361 | Notwithstanding any other provision of this License, for material you 362 | add to a covered work, you may (if authorized by the copyright holders of 363 | that material) supplement the terms of this License with terms: 364 | 365 | a) Disclaiming warranty or limiting liability differently from the 366 | terms of sections 15 and 16 of this License; or 367 | 368 | b) Requiring preservation of specified reasonable legal notices or 369 | author attributions in that material or in the Appropriate Legal 370 | Notices displayed by works containing it; or 371 | 372 | c) Prohibiting misrepresentation of the origin of that material, or 373 | requiring that modified versions of such material be marked in 374 | reasonable ways as different from the original version; or 375 | 376 | d) Limiting the use for publicity purposes of names of licensors or 377 | authors of the material; or 378 | 379 | e) Declining to grant rights under trademark law for use of some 380 | trade names, trademarks, or service marks; or 381 | 382 | f) Requiring indemnification of licensors and authors of that 383 | material by anyone who conveys the material (or modified versions of 384 | it) with contractual assumptions of liability to the recipient, for 385 | any liability that these contractual assumptions directly impose on 386 | those licensors and authors. 387 | 388 | All other non-permissive additional terms are considered "further 389 | restrictions" within the meaning of section 10. If the Program as you 390 | received it, or any part of it, contains a notice stating that it is 391 | governed by this License along with a term that is a further 392 | restriction, you may remove that term. If a license document contains 393 | a further restriction but permits relicensing or conveying under this 394 | License, you may add to a covered work material governed by the terms 395 | of that license document, provided that the further restriction does 396 | not survive such relicensing or conveying. 397 | 398 | If you add terms to a covered work in accord with this section, you 399 | must place, in the relevant source files, a statement of the 400 | additional terms that apply to those files, or a notice indicating 401 | where to find the applicable terms. 402 | 403 | Additional terms, permissive or non-permissive, may be stated in the 404 | form of a separately written license, or stated as exceptions; 405 | the above requirements apply either way. 406 | 407 | 8. Termination. 408 | 409 | You may not propagate or modify a covered work except as expressly 410 | provided under this License. Any attempt otherwise to propagate or 411 | modify it is void, and will automatically terminate your rights under 412 | this License (including any patent licenses granted under the third 413 | paragraph of section 11). 414 | 415 | However, if you cease all violation of this License, then your 416 | license from a particular copyright holder is reinstated (a) 417 | provisionally, unless and until the copyright holder explicitly and 418 | finally terminates your license, and (b) permanently, if the copyright 419 | holder fails to notify you of the violation by some reasonable means 420 | prior to 60 days after the cessation. 421 | 422 | Moreover, your license from a particular copyright holder is 423 | reinstated permanently if the copyright holder notifies you of the 424 | violation by some reasonable means, this is the first time you have 425 | received notice of violation of this License (for any work) from that 426 | copyright holder, and you cure the violation prior to 30 days after 427 | your receipt of the notice. 428 | 429 | Termination of your rights under this section does not terminate the 430 | licenses of parties who have received copies or rights from you under 431 | this License. If your rights have been terminated and not permanently 432 | reinstated, you do not qualify to receive new licenses for the same 433 | material under section 10. 434 | 435 | 9. Acceptance Not Required for Having Copies. 436 | 437 | You are not required to accept this License in order to receive or 438 | run a copy of the Program. Ancillary propagation of a covered work 439 | occurring solely as a consequence of using peer-to-peer transmission 440 | to receive a copy likewise does not require acceptance. However, 441 | nothing other than this License grants you permission to propagate or 442 | modify any covered work. These actions infringe copyright if you do 443 | not accept this License. Therefore, by modifying or propagating a 444 | covered work, you indicate your acceptance of this License to do so. 445 | 446 | 10. Automatic Licensing of Downstream Recipients. 447 | 448 | Each time you convey a covered work, the recipient automatically 449 | receives a license from the original licensors, to run, modify and 450 | propagate that work, subject to this License. You are not responsible 451 | for enforcing compliance by third parties with this License. 452 | 453 | An "entity transaction" is a transaction transferring control of an 454 | organization, or substantially all assets of one, or subdividing an 455 | organization, or merging organizations. If propagation of a covered 456 | work results from an entity transaction, each party to that 457 | transaction who receives a copy of the work also receives whatever 458 | licenses to the work the party's predecessor in interest had or could 459 | give under the previous paragraph, plus a right to possession of the 460 | Corresponding Source of the work from the predecessor in interest, if 461 | the predecessor has it or can get it with reasonable efforts. 462 | 463 | You may not impose any further restrictions on the exercise of the 464 | rights granted or affirmed under this License. For example, you may 465 | not impose a license fee, royalty, or other charge for exercise of 466 | rights granted under this License, and you may not initiate litigation 467 | (including a cross-claim or counterclaim in a lawsuit) alleging that 468 | any patent claim is infringed by making, using, selling, offering for 469 | sale, or importing the Program or any portion of it. 470 | 471 | 11. Patents. 472 | 473 | A "contributor" is a copyright holder who authorizes use under this 474 | License of the Program or a work on which the Program is based. The 475 | work thus licensed is called the contributor's "contributor version". 476 | 477 | A contributor's "essential patent claims" are all patent claims 478 | owned or controlled by the contributor, whether already acquired or 479 | hereafter acquired, that would be infringed by some manner, permitted 480 | by this License, of making, using, or selling its contributor version, 481 | but do not include claims that would be infringed only as a 482 | consequence of further modification of the contributor version. For 483 | purposes of this definition, "control" includes the right to grant 484 | patent sublicenses in a manner consistent with the requirements of 485 | this License. 486 | 487 | Each contributor grants you a non-exclusive, worldwide, royalty-free 488 | patent license under the contributor's essential patent claims, to 489 | make, use, sell, offer for sale, import and otherwise run, modify and 490 | propagate the contents of its contributor version. 491 | 492 | In the following three paragraphs, a "patent license" is any express 493 | agreement or commitment, however denominated, not to enforce a patent 494 | (such as an express permission to practice a patent or covenant not to 495 | sue for patent infringement). To "grant" such a patent license to a 496 | party means to make such an agreement or commitment not to enforce a 497 | patent against the party. 498 | 499 | If you convey a covered work, knowingly relying on a patent license, 500 | and the Corresponding Source of the work is not available for anyone 501 | to copy, free of charge and under the terms of this License, through a 502 | publicly available network server or other readily accessible means, 503 | then you must either (1) cause the Corresponding Source to be so 504 | available, or (2) arrange to deprive yourself of the benefit of the 505 | patent license for this particular work, or (3) arrange, in a manner 506 | consistent with the requirements of this License, to extend the patent 507 | license to downstream recipients. "Knowingly relying" means you have 508 | actual knowledge that, but for the patent license, your conveying the 509 | covered work in a country, or your recipient's use of the covered work 510 | in a country, would infringe one or more identifiable patents in that 511 | country that you have reason to believe are valid. 512 | 513 | If, pursuant to or in connection with a single transaction or 514 | arrangement, you convey, or propagate by procuring conveyance of, a 515 | covered work, and grant a patent license to some of the parties 516 | receiving the covered work authorizing them to use, propagate, modify 517 | or convey a specific copy of the covered work, then the patent license 518 | you grant is automatically extended to all recipients of the covered 519 | work and works based on it. 520 | 521 | A patent license is "discriminatory" if it does not include within 522 | the scope of its coverage, prohibits the exercise of, or is 523 | conditioned on the non-exercise of one or more of the rights that are 524 | specifically granted under this License. You may not convey a covered 525 | work if you are a party to an arrangement with a third party that is 526 | in the business of distributing software, under which you make payment 527 | to the third party based on the extent of your activity of conveying 528 | the work, and under which the third party grants, to any of the 529 | parties who would receive the covered work from you, a discriminatory 530 | patent license (a) in connection with copies of the covered work 531 | conveyed by you (or copies made from those copies), or (b) primarily 532 | for and in connection with specific products or compilations that 533 | contain the covered work, unless you entered into that arrangement, 534 | or that patent license was granted, prior to 28 March 2007. 535 | 536 | Nothing in this License shall be construed as excluding or limiting 537 | any implied license or other defenses to infringement that may 538 | otherwise be available to you under applicable patent law. 539 | 540 | 12. No Surrender of Others' Freedom. 541 | 542 | If conditions are imposed on you (whether by court order, agreement or 543 | otherwise) that contradict the conditions of this License, they do not 544 | excuse you from the conditions of this License. If you cannot convey a 545 | covered work so as to satisfy simultaneously your obligations under this 546 | License and any other pertinent obligations, then as a consequence you may 547 | not convey it at all. For example, if you agree to terms that obligate you 548 | to collect a royalty for further conveying from those to whom you convey 549 | the Program, the only way you could satisfy both those terms and this 550 | License would be to refrain entirely from conveying the Program. 551 | 552 | 13. Use with the GNU Affero General Public License. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU Affero General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the special requirements of the GNU Affero General Public License, 560 | section 13, concerning interaction through a network will apply to the 561 | combination as such. 562 | 563 | 14. Revised Versions of this License. 564 | 565 | The Free Software Foundation may publish revised and/or new versions of 566 | the GNU General Public License from time to time. Such new versions will 567 | be similar in spirit to the present version, but may differ in detail to 568 | address new problems or concerns. 569 | 570 | Each version is given a distinguishing version number. If the 571 | Program specifies that a certain numbered version of the GNU General 572 | Public License "or any later version" applies to it, you have the 573 | option of following the terms and conditions either of that numbered 574 | version or of any later version published by the Free Software 575 | Foundation. If the Program does not specify a version number of the 576 | GNU General Public License, you may choose any version ever published 577 | by the Free Software Foundation. 578 | 579 | If the Program specifies that a proxy can decide which future 580 | versions of the GNU General Public License can be used, that proxy's 581 | public statement of acceptance of a version permanently authorizes you 582 | to choose that version for the Program. 583 | 584 | Later license versions may give you additional or different 585 | permissions. However, no additional obligations are imposed on any 586 | author or copyright holder as a result of your choosing to follow a 587 | later version. 588 | 589 | 15. Disclaimer of Warranty. 590 | 591 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 592 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 596 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 597 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 599 | 600 | 16. Limitation of Liability. 601 | 602 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 610 | SUCH DAMAGES. 611 | 612 | 17. Interpretation of Sections 15 and 16. 613 | 614 | If the disclaimer of warranty and limitation of liability provided 615 | above cannot be given local legal effect according to their terms, 616 | reviewing courts shall apply local law that most closely approximates 617 | an absolute waiver of all civil liability in connection with the 618 | Program, unless a warranty or assumption of liability accompanies a 619 | copy of the Program in return for a fee. 620 | 621 | END OF TERMS AND CONDITIONS 622 | 623 | How to Apply These Terms to Your New Programs 624 | 625 | If you develop a new program, and you want it to be of the greatest 626 | possible use to the public, the best way to achieve this is to make it 627 | free software which everyone can redistribute and change under these terms. 628 | 629 | To do so, attach the following notices to the program. It is safest 630 | to attach them to the start of each source file to most effectively 631 | state the exclusion of warranty; and each file should have at least 632 | the "copyright" line and a pointer to where the full notice is found. 633 | 634 | 635 | Copyright (C) 636 | 637 | This program is free software: you can redistribute it and/or modify 638 | it under the terms of the GNU General Public License as published by 639 | the Free Software Foundation, either version 3 of the License, or 640 | (at your option) any later version. 641 | 642 | This program is distributed in the hope that it will be useful, 643 | but WITHOUT ANY WARRANTY; without even the implied warranty of 644 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 645 | GNU General Public License for more details. 646 | 647 | You should have received a copy of the GNU General Public License 648 | along with this program. If not, see . 649 | 650 | Also add information on how to contact you by electronic and paper mail. 651 | 652 | If the program does terminal interaction, make it output a short 653 | notice like this when it starts in an interactive mode: 654 | 655 | Copyright (C) 656 | This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 657 | This is free software, and you are welcome to redistribute it 658 | under certain conditions; type `show c' for details. 659 | 660 | The hypothetical commands `show w' and `show c' should show the appropriate 661 | parts of the General Public License. Of course, your program's commands 662 | might be different; for a GUI interface, you would use an "about box". 663 | 664 | You should also get your employer (if you work as a programmer) or school, 665 | if any, to sign a "copyright disclaimer" for the program, if necessary. 666 | For more information on this, and how to apply and follow the GNU GPL, see 667 | . 668 | 669 | The GNU General Public License does not permit incorporating your program 670 | into proprietary programs. If your program is a subroutine library, you 671 | may consider it more useful to permit linking proprietary applications with 672 | the library. If this is what you want to do, use the GNU Lesser General 673 | Public License instead of this License. But first, please read 674 | . 675 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Compiling Python into AWS Lambda / Step Function 2 | 3 | Ben North 4 | ([GitHub](https://www.github.com/bennorth/) 5 | / [blog](http://www.redfrontdoor.org/blog/)), 6 | March 2018 7 | [(repository root)](https://github.com/bennorth/pyawssfn) 8 | 9 | 10 | ## Installation 11 | 12 | ```bash 13 | pip install . 14 | ``` 15 | 16 | 17 | # Background 18 | 19 | Among the components of Amazon Web Services are the following two 20 | parts of their 'serverless' approach: 21 | 22 | * [Lambda](https://aws.amazon.com/lambda/) — a 'Lambda function' 23 | is a self-contained piece of code which AWS runs on your behalf in 24 | response to triggers you specify; 25 | * [Step Functions](https://aws.amazon.com/step-functions/) — a 26 | 'Step Function' is a mechanism for controlling the interlinked 27 | operation of multiple steps, including invocation of Lambda 28 | functions. 29 | 30 | While Lambda functions can be written in many languages, to write a 31 | Step Function you describe the logic as a state machine in JSON. This 32 | seems cumbersome when compared to our normal way of describing how to 33 | control interlinked computations, which is to write some Python (or 34 | C#, or Java, or...). 35 | 36 | Based on this observation, the tools presented here are a 37 | 'plausibility argument of concept' for the idea that you could write 38 | your top-level logic as a Python program and have it compiled into a 39 | Step Function state machine. (I haven't developed this far enough to 40 | call it a '*proof* of concept'.) 41 | 42 | One of the desired properties of the system is that the source program 43 | should be more or less 'normal Python'. It should be possible to use 44 | it in two ways: 45 | 46 | * Run it as a Python program with the usual Python interpreter; 47 | * Compile it into a Step Function and run in the AWS cloud. 48 | 49 | The ability to run your logic as a normal Python program allows local 50 | development and testing. 51 | 52 | 53 | # Status 54 | 55 | Although I think the tools here do show that the idea has promise, 56 | there would be [plenty still to do](future-work.md) to make them 57 | useful for production purposes. I am very unlikely to have time in 58 | the near future to develop this any further, but the source is all 59 | here (under GPL) if anybody wants to build on it. 60 | 61 | 62 | # General approach 63 | 64 | ## Compile Python code to Step Function state machine 65 | 66 | The ['Python to Step Function compiler' tool](src/pysfn/tools/compile.py), 67 | `pysfn.tools.compile`, 68 | reads in a file of Python code and emits JSON corresponding to the 69 | control flow of a specified 'entry point' function in that code. The 70 | resulting JSON is used for the creation of an AWS Step Function. 71 | Various supplied Python functions allow the programmer to express 72 | intent in terms of retry characteristics, parallel invocations, error 73 | handling, etc. Nonetheless the code is valid normal Python and 74 | executes with (mostly) equivalent semantics to those the resulting 75 | Step Function will have. 76 | 77 | ## Wrap original Python code as Lambda function 78 | 79 | The ['Python to Step Function wrapper compiler' tool]( 80 | src/pysfn/tools/gen_lambda.py), `pysfn.tools.gen_lambda`, 81 | constructs a zip-file containing the original Python 82 | code together with a small wrapper. The zip-file is suitable for 83 | uploading as an AWS Lambda function. This gives the top-level Step 84 | Function access to what were callees in the original Python code. 85 | 86 | 87 | # Example 88 | 89 | In the below I have omitted details like creation of 90 | [IAM users](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html), 91 | creation of 92 | [roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html), 93 | etc. See Amazon's documentation on these points. 94 | 95 | 96 | ## Run unit tests on original Python 97 | 98 | The [original Python source](examples/analyse_text.py) consists of a 99 | main 100 | driver function, with a collection of small functions used by the main 101 | function. It is very simple, performing a few computations on an 102 | input string, but serves the purpose of illustrating the compilation 103 | process. It has a suite of unit tests: 104 | 105 | ```bash 106 | pip install .[dev] # install development dependencies 107 | pytest tests/test_analyse_text.py 108 | ``` 109 | 110 | Output: 111 | ``` 112 | # ... ======== 10 passed in 0.02 seconds ======== ... 113 | ``` 114 | 115 | ## Wrap original Python ready for Lambda 116 | 117 | ```bash 118 | python -m pysfn.tools.gen_lambda examples/analyse_text.py lambda-function.zip 119 | unzip -l lambda-function.zip 120 | ``` 121 | 122 | Output: 123 | ``` 124 | Archive: lambda-function.zip 125 | Length Date Time Name 126 | --------- ---------- ----- ---- 127 | 302 1980-01-01 00:00 handler.py 128 | 1981 2018-03-25 20:01 inner/analyse_text.py 129 | 452 2018-03-23 22:32 pysfn.py 130 | --------- ------- 131 | 2735 3 files 132 | ``` 133 | 134 | Now upload `lambda-function.zip` as a new Lambda function with the 135 | `Python 3.6` runtime, specify `handler.dispatch` as its entry point, 136 | and note its ARN for use in the next step. 137 | 138 | ## Compile original Python into Step Function JSON 139 | 140 | ```bash 141 | python -m pysfn.tools.compile examples/analyse_text.py LAMBDA-FUN-ARN > examples/stepfun.json 142 | cat examples/stepfun.json 143 | ``` 144 | 145 | Output (the [full output](examples/stepfun.json) is 196 lines): 146 | ``` 147 | { 148 | "States": { 149 | "n0": { 150 | "Type": "Pass", 151 | "Result": { 152 | "function": "get_summary", 153 | "arg_names": [ 154 | "text" 155 | ] 156 | }, 157 | "ResultPath": "$.call_descr", 158 | "Next": "n1" 159 | }, 160 | 161 | [...] 162 | 163 | "n19": { 164 | "Type": "Succeed", 165 | "InputPath": "$.locals.result" 166 | } 167 | }, 168 | "StartAt": "n0" 169 | } 170 | ``` 171 | 172 | Now copy-and-paste this as the JSON for a new Step Function. 173 | 174 | ## Execute Step Function 175 | 176 | You should now be able to perform an execution of this Step Function with, 177 | for example, the input 178 | ```json 179 | { 180 | "locals": { 181 | "text": "a short example" 182 | } 183 | } 184 | ``` 185 | to get the output 186 | ```json 187 | { 188 | "output": "text starts with a, has 15 chars, 5 vowels, and 2 spaces" 189 | } 190 | ``` 191 | 192 | 193 | # More documentation 194 | 195 | * [Implementation notes](implementation-notes.md) 196 | * [Future work](future-work.md) 197 | 198 | 199 | --- 200 | 201 | This document: Copyright 2018 Ben North; licensed under 202 | [CC BY-SA 4.0](http://creativecommons.org/licenses/by-sa/4.0/) 203 | 204 | See the file `COPYING` for full licensing details. 205 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.0.0a0.dev0 2 | -------------------------------------------------------------------------------- /examples/analyse_text.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Ben North 2 | # 3 | # This file is part of 'plausibility argument of concept for compiling 4 | # Python into Amazon Step Function state machine JSON'. 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | 20 | import pysfn as PSF 21 | 22 | 23 | class TextTooShortError(Exception): 24 | pass 25 | 26 | 27 | def get_summary(text): 28 | if len(text) == 0: 29 | raise TextTooShortError 30 | return {'head': text[0]} 31 | 32 | 33 | def augment_summary(text, summary): 34 | aug_summary = dict(summary) 35 | aug_summary['n_characters'] = len(text) 36 | return aug_summary 37 | 38 | 39 | def get_n_vowels(text): 40 | return sum(text.count(v) for v in 'aeiou') 41 | 42 | 43 | def get_n_spaces(text): 44 | return text.count(' ') 45 | 46 | 47 | def format_result(summary, infos): 48 | """ 49 | Expect inputs of: 50 | 51 | summary --- dict with 'head' and 'n_characters' as keys 52 | infos --- list of two elements, each a number 53 | """ 54 | return (f'text starts with {summary["head"]},' 55 | f' has {summary["n_characters"]} chars,' 56 | f' {infos[0]} vowels, and' 57 | f' {infos[1]} spaces') 58 | 59 | 60 | def format_c_result(summary): 61 | return f'text starts with "c"; look: "{summary["head"]}"' 62 | 63 | 64 | # Top-level function, very loosely inspired by the control flow in 65 | # 66 | # https://github.com/aws-samples/lambda-refarch-imagerecognition 67 | # 68 | @PSF.main 69 | def summarise(text): 70 | try: 71 | summary = get_summary(text) 72 | except TextTooShortError: 73 | raise PSF.Fail('MalformedText', 'text too short') 74 | 75 | if (PSF.StringEquals(summary['head'], 'a') 76 | or PSF.StringEquals(summary['head'], 'b')): 77 | summary = PSF.with_retry_spec(augment_summary, (text, summary), 78 | (['States.ALL'], 1, 2, 1.5)) 79 | 80 | def get_n_vowels_task(): 81 | result = get_n_vowels(text) 82 | return result 83 | # 84 | def get_n_spaces_task(): 85 | result = get_n_spaces(text) 86 | return result 87 | # 88 | more_info = PSF.parallel(get_n_vowels_task, get_n_spaces_task) 89 | 90 | result = format_result(summary, more_info) 91 | # 92 | elif PSF.StringEquals(summary['head'], 'c'): 93 | result = format_c_result(summary) 94 | # 95 | else: 96 | raise PSF.Fail('MalformedText', 'wrong starting letter') 97 | 98 | return result 99 | -------------------------------------------------------------------------------- /examples/stepfun.json: -------------------------------------------------------------------------------- 1 | { 2 | "States": { 3 | "n0": { 4 | "Type": "Pass", 5 | "Result": { 6 | "function": "get_summary", 7 | "arg_names": [ 8 | "text" 9 | ] 10 | }, 11 | "ResultPath": "$.call_descr", 12 | "Next": "n1" 13 | }, 14 | "n1": { 15 | "Type": "Task", 16 | "Resource": "LAMBDA-FUN-ARN", 17 | "ResultPath": "$.locals.summary", 18 | "Catch": [ 19 | { 20 | "ErrorEquals": [ 21 | "TextTooShortError" 22 | ], 23 | "Next": "n2" 24 | } 25 | ], 26 | "Next": "n18" 27 | }, 28 | "n2": { 29 | "Type": "Fail", 30 | "Error": "MalformedText", 31 | "Cause": "text too short" 32 | }, 33 | "n18": { 34 | "Type": "Choice", 35 | "Choices": [ 36 | { 37 | "Or": [ 38 | { 39 | "Variable": "$.locals.summary.head", 40 | "StringEquals": "a" 41 | }, 42 | { 43 | "Variable": "$.locals.summary.head", 44 | "StringEquals": "b" 45 | } 46 | ], 47 | "Next": "n3" 48 | } 49 | ], 50 | "Default": "n17" 51 | }, 52 | "n3": { 53 | "Type": "Pass", 54 | "Result": { 55 | "function": "augment_summary", 56 | "arg_names": [ 57 | "text", 58 | "summary" 59 | ] 60 | }, 61 | "ResultPath": "$.call_descr", 62 | "Next": "n4" 63 | }, 64 | "n4": { 65 | "Type": "Task", 66 | "Resource": "LAMBDA-FUN-ARN", 67 | "ResultPath": "$.locals.summary", 68 | "Retry": [ 69 | { 70 | "ErrorEquals": [ 71 | "States.ALL" 72 | ], 73 | "IntervalSeconds": 1, 74 | "MaxAttempts": 2, 75 | "BackoffRate": 1.5 76 | } 77 | ], 78 | "Next": "n11" 79 | }, 80 | "n11": { 81 | "Type": "Parallel", 82 | "Branches": [ 83 | { 84 | "States": { 85 | "n5": { 86 | "Type": "Pass", 87 | "Result": { 88 | "function": "get_n_vowels", 89 | "arg_names": [ 90 | "text" 91 | ] 92 | }, 93 | "ResultPath": "$.call_descr", 94 | "Next": "n6" 95 | }, 96 | "n6": { 97 | "Type": "Task", 98 | "Resource": "LAMBDA-FUN-ARN", 99 | "ResultPath": "$.locals.result", 100 | "Next": "n7" 101 | }, 102 | "n7": { 103 | "Type": "Succeed", 104 | "InputPath": "$.locals.result" 105 | } 106 | }, 107 | "StartAt": "n5" 108 | }, 109 | { 110 | "States": { 111 | "n8": { 112 | "Type": "Pass", 113 | "Result": { 114 | "function": "get_n_spaces", 115 | "arg_names": [ 116 | "text" 117 | ] 118 | }, 119 | "ResultPath": "$.call_descr", 120 | "Next": "n9" 121 | }, 122 | "n9": { 123 | "Type": "Task", 124 | "Resource": "LAMBDA-FUN-ARN", 125 | "ResultPath": "$.locals.result", 126 | "Next": "n10" 127 | }, 128 | "n10": { 129 | "Type": "Succeed", 130 | "InputPath": "$.locals.result" 131 | } 132 | }, 133 | "StartAt": "n8" 134 | } 135 | ], 136 | "ResultPath": "$.locals.more_info", 137 | "Next": "n12" 138 | }, 139 | "n12": { 140 | "Type": "Pass", 141 | "Result": { 142 | "function": "format_result", 143 | "arg_names": [ 144 | "summary", 145 | "more_info" 146 | ] 147 | }, 148 | "ResultPath": "$.call_descr", 149 | "Next": "n13" 150 | }, 151 | "n13": { 152 | "Type": "Task", 153 | "Resource": "LAMBDA-FUN-ARN", 154 | "ResultPath": "$.locals.result", 155 | "Next": "n19" 156 | }, 157 | "n17": { 158 | "Type": "Choice", 159 | "Choices": [ 160 | { 161 | "Variable": "$.locals.summary.head", 162 | "StringEquals": "c", 163 | "Next": "n14" 164 | } 165 | ], 166 | "Default": "n16" 167 | }, 168 | "n14": { 169 | "Type": "Pass", 170 | "Result": { 171 | "function": "format_c_result", 172 | "arg_names": [ 173 | "summary" 174 | ] 175 | }, 176 | "ResultPath": "$.call_descr", 177 | "Next": "n15" 178 | }, 179 | "n15": { 180 | "Type": "Task", 181 | "Resource": "LAMBDA-FUN-ARN", 182 | "ResultPath": "$.locals.result", 183 | "Next": "n19" 184 | }, 185 | "n16": { 186 | "Type": "Fail", 187 | "Error": "MalformedText", 188 | "Cause": "wrong starting letter" 189 | }, 190 | "n19": { 191 | "Type": "Succeed", 192 | "InputPath": "$.locals.result" 193 | } 194 | }, 195 | "StartAt": "n0" 196 | } 197 | -------------------------------------------------------------------------------- /future-work.md: -------------------------------------------------------------------------------- 1 | # Future work 2 | 3 | Ben North 4 | ([GitHub](https://www.github.com/bennorth/) 5 | / [blog](http://www.redfrontdoor.org/blog/)), 6 | March 2018 7 | [(repository root)](https://github.com/bennorth/pyawssfn) 8 | 9 | 10 | # Small self-contained ideas 11 | 12 | * Implement remainder of choice-rule predicates. Currently only 13 | string predicates are handled, because we access the `s` attribute 14 | of the second argument. 15 | 16 | * Implement remaining predicate combinator, `Not`. 17 | 18 | * Allow list indexing in main function. Currently only dictionary 19 | lookup works, but should be easy enough to also allow things like 20 | `PSF.StringEquals(things[7], 'hello')`. 21 | 22 | * Could handle bare `return` by converting to `return None` and 23 | thence to a `Succeed` with `InputPath=null`. 24 | 25 | * More-helpful exceptions if `RetrySpecIR` given a node of the wrong 26 | form; test for these situations. 27 | 28 | * More-helpful exceptions if `Catcher` given a node of the wrong 29 | form; test for these situations. 30 | 31 | * When building `TryIR`, check body is single assignment. If not, 32 | could maybe convert to `Parallel` with just one strand, then extract 33 | single result? 34 | 35 | * Notice and collapse `if`/`elif`/`elif`/`else` chains into one 36 | `Choice` state. 37 | 38 | * Check that local definitions used for `Parallel` states have no 39 | args. 40 | 41 | * Check for unused or undefined branches of a `Parallel` state. 42 | 43 | * Allow `Parallel` state to have `Retry` and `Catch` clauses. In 44 | Python, the latter is 'allow `PSF.parallel()` inside 45 | `try`/`except`'. 46 | 47 | * Proper nested scopes for local variables of `Parallel` sub-tasks. 48 | 49 | * Provide defaults for retry-spec tuples, in line with JSON. 50 | 51 | * Allow keyword arguments in a `FunctionCallIR`. 52 | 53 | * Allow `if` without `else`. Will be mildly fiddly because our 54 | concept of connecting up the 'next state' can't currently reach 55 | inside the `Choice` state to set its `Default` field. Could 56 | possibly replace `exit_states` with a collection of closures which 57 | know how to set the correct field of the correct object? 58 | Alternatively, always create an `else` branch at the State Machine 59 | level, consisting of a single no-op `Pass` state. 60 | 61 | * Validate final state machine; e.g., there should be no unexpected 62 | states with un-filled 'next state' slots. 63 | 64 | * Better and more thorough error-handling throughout, including 65 | more-helpful error messages when requirements are not met. 66 | 67 | * Tools to automatically deploy Step Function and Lambda. 68 | 69 | * Detection of tests like in `if x == 'JPEG'`, and conversion into 70 | equivalent use of `StringEquals`. 71 | 72 | * Avoid having to ferry entire state back/forth to the Lambda 73 | machinery when only the function args and its return value actually 74 | need to be communicated. 75 | 76 | * Implement `Wait` state. Could be as simple as noticing a magic 77 | function `PSF.Wait(...)`. Or could translate Python `time.sleep()` 78 | into `Wait`. 79 | 80 | * Special entry state to extract fields from input corresponding to 81 | function parameter names, and create an initial `$.locals`. 82 | 83 | * Allow use of functions called only for side-effect. (I.e., just 84 | `foo()` not `x = bar(y)`.) 85 | 86 | 87 | # Higher-level research avenues 88 | 89 | ## Automatic parallelisation 90 | 91 | Automatic deduction, based on data-flow, of which operations are 92 | independent and could be gathered into a `Parallel` state. Some care 93 | needed because there might be hidden dependencies: One function 94 | invocation might have some side-effect that the next computation 95 | relies on, without this being explicit in the data-flow through 96 | variables. E.g., in the snippet 97 | 98 | ```python 99 | c = foo(a) 100 | b = bar(a) 101 | ``` 102 | 103 | it seems that `foo(a)` and `bar(a)` can proceed independently, in 104 | parallel, but perhaps `bar(a)` relies on some global state which 105 | `foo(a)` establishes, like a change to a shared database. 106 | 107 | ## Directly interpret Python 108 | 109 | The state-machine runtime could effectively perform the compilation 110 | work itself, directly understanding Python. 111 | 112 | ## Higher-level serialisation/deserialisation 113 | 114 | Currently everything has to be a JSON-friendly object (number, string, 115 | list/array, dictionary). Passing objects more complex than this 116 | around between invocations of Lambda functions would need some 117 | thought. For some situations it might be enough to add type 118 | annotations to the various functions and insert serialisation and 119 | deserialisation code into the 'wrapper'. For big objects some level 120 | of indirection to/from an external object store (e.g., S3 buckets) 121 | might be required. 122 | 123 | ## Extract in-line computation from top-level function 124 | 125 | If some part of the top-level computation is a short sequence of 126 | statements, it would be convenient for the translation tool to do the 127 | equivalent of an 'extract method' refactoring, make the extracted code 128 | accessible via the Lambda, and replace it with a call. 129 | 130 | ## Loops 131 | 132 | It is entirely possible to have loops in a Step Function state 133 | machine. On the Python level, it would be convenient to be able to 134 | use the familiar 135 | 136 | ```python 137 | for x in xs: 138 | do_stuff(x) 139 | ``` 140 | 141 | syntax. Creation of a state-machine graph containing a cycle would be 142 | fairly straightforward, but we would also require a JSON-friendly 143 | iterator. Initially, loops could be restricted just to iterating over 144 | ists or dicts. 145 | 146 | ## Dependencies 147 | 148 | It is likely that any non-trivial application of these ideas would 149 | involve the use of external libraries. It would be convenient if the 150 | tool could automatically detect dependencies, and bundle them up into 151 | the zip-file used to create the Lambda. A solution requiring 152 | something along the lines of a `requirements.txt` might be a 153 | reasonable halfway house. Could be combined with the 'deployment 154 | tools' thought, looking at things like [Zappa](https://www.zappa.io/) 155 | for ideas. 156 | 157 | ## Integration with `git` 158 | 159 | And most likely GitHub in particular. 160 | 161 | ## Conversion of unit tests 162 | 163 | The original Python code presumably has unit tests. It would be 164 | convenient if these could be converted into tests which performed 165 | invocations of the Step Function. 166 | 167 | ## Optimise resulting state machine 168 | 169 | AWS bills Step Function invocations according to how many transitions 170 | the state machine makes. Could the structure be optimised so as to 171 | minimise the expected number of transitions, under appropriate 172 | assumptions for what the input is likely to look like? 173 | 174 | 175 | # Wider-scope questions 176 | 177 | The following questions are not strictly within the scope of a tool 178 | which translates Python code to a Step Function, but arose while doing 179 | the work: 180 | 181 | ## Rethink `Parallel` state 182 | 183 | Reconsider the current design whereby the branches of a `Parallel` are 184 | self-contained state machines. It seems like it would be possible to 185 | have each branch consist just of an entry-point state-name, within the 186 | same top-level collection of states. This could simplify the task of 187 | translating programming languages to state machines, and (as a small 188 | side-benefit) allow re-use of states between top-level execution and 189 | parallel-branch execution. 190 | 191 | ## Automatically learn appropriate retry specifications 192 | 193 | Could the programmer be freed from having to specify appropriate retry 194 | specifications? If all Lambda invocations were pure (or, more weakly, 195 | 'idempotent' might be enough), then the runtime could gather 196 | statistics on each Lambda's reliability, how failures tend to be 197 | clustered temporally, etc., and deduce suitable retry specs. 198 | 199 | ## Similar tooling for Apache Airflow 200 | 201 | [Airflow](https://github.com/apache/incubator-airflow) allows the 202 | construction of workflow DAGs in code. There might be scope to write 203 | a compiler, similar to the one in this repo, to extract the data-flow 204 | from Python code and convert it into an Airflow DAG. 205 | 206 | 207 | --- 208 | 209 | This document: Copyright 2018 Ben North; licensed under 210 | [CC BY-SA 4.0](http://creativecommons.org/licenses/by-sa/4.0/) 211 | 212 | See the file `COPYING` for full licensing details. 213 | -------------------------------------------------------------------------------- /implementation-notes.md: -------------------------------------------------------------------------------- 1 | # Implementation notes 2 | 3 | Ben North 4 | ([GitHub](https://www.github.com/bennorth/) 5 | / [blog](http://www.redfrontdoor.org/blog/)), 6 | March 2018 7 | [(repository root)](https://github.com/bennorth/pyawssfn) 8 | 9 | 10 | # Intermediate Representation 11 | 12 | Various concepts within the State Machine have Python representations 13 | at a level suitable for working with in Python. I.e., they have 14 | Python attributes corresonding to their parts. These are: 15 | 16 | ## `ChoiceCondition` 17 | 18 | In fact, an instance of one of the two derived classes 19 | 20 | * `TestComparison` 21 | * `TestCombinator` 22 | 23 | Represents what will be one entry in a `Choice` state's `Choices` 24 | slot. It may or may not have a `Next` slot, depending on whether it 25 | is part of a larger condition. 26 | 27 | ## `ReturnIR` 28 | 29 | Represents a `return some_variable` statement. Can only return a 30 | variable. 31 | 32 | ## `RaiseIR` 33 | 34 | Represents a `raise PSF.Fail("BadThing", "something went wrong")` 35 | statement. The exception raised must be of that form. 36 | 37 | ## `FunctionCallIR` 38 | 39 | Represents a function call of one of the two forms 40 | 41 | * `foo(bar, baz)` 42 | * `PSF.with_retry_spec(foo, (bar, baz), spec1, spec2)` 43 | 44 | In both of these examples, the function-name is `foo` and the argnames 45 | list is `['bar', 'baz']`. The second example also has a retry-spec. 46 | 47 | ## `AssignmentSourceIR` 48 | 49 | Represents the source of an assignment; e.g., in `foo = bar(baz)`, the 50 | assignment source is the function call `bar(baz)`. 51 | 52 | ## `AssignmentIR` 53 | 54 | Represents an assignment to a single simple variable from a source. 55 | 56 | ## `SuiteIR` 57 | 58 | Represents a Python suite of statements, e.g., the body executed if 59 | the test in an `if` statement evaluates to true. 60 | 61 | ## `CatcherIR` 62 | 63 | Represents one `except` clause within a Python `try`/`except` 64 | statement. The Python-level exception name gets mapped by the Step 65 | Function machinery into the Error Name. 66 | 67 | ## `TryIR` 68 | 69 | Represents a `try`/`except` statement, with body and some handlers 70 | (which become `CatcherIR`s). In a Step Function, catchers apply to a 71 | `Task`, so the body can be only a single assignment. 72 | 73 | ## `IfIR` 74 | 75 | Represents an `if`/`else` statement, with test (`ChoiceCondition`), 76 | body for when test true, and body for when test false. 77 | 78 | ## `ParallelIR` 79 | 80 | Represents a `Parallel` state. A few plausible approaches to how the 81 | Python should look for this; settled on local function definitions, 82 | which have access to variables in the enclosing scope. See 83 | `sample_parallel_invocation` test fixture and its usage. Seems 84 | slightly clunky to pass around the 'enclosing scope' of definitions 85 | but not too bad. 86 | 87 | There is precedent for the `(function, (arg1, arg2))` description of a 88 | function in, for example, `multiprocessing.Process()`. 89 | 90 | 91 | # Local variables as Step Function state 92 | 93 | The local variables which exist within the 'main' function will be 94 | stored in the Step Function's state object under a `locals` 95 | sub-object. E.g., the local variable `foo` will be serialised into 96 | the JSON sub-object `locals.foo`. 97 | 98 | For these purposes, the parameters of the function are treated as 99 | local variables. E.g., a function with a parameter `height` will give 100 | rise to a sub-object `locals.height`. 101 | 102 | Python objects which are dictionaries have access to their (chained) 103 | keys converted into JSON sub-object access. E.g., the Python chained 104 | key lookup `foo['bar']['baz']` will be converted to the JSON 105 | expression `foo.bar.baz`. 106 | 107 | 108 | # State Machine Representation 109 | 110 | The concepts also have Python representations essentially equivalent 111 | to what will be written out as the JSON description of the state 112 | machine. Often this will be a dictionary, although lists and literals 113 | also occur. 114 | 115 | ## `ChoiceCondition` 116 | 117 | Objects of (one of the two subclasses of) this class have a method 118 | `as_choice_rule_smr(next)` which returns a dictionary suitable for use 119 | as one element of the `Choices` list of a `Choice` state, or as a 120 | component of such an element. Top-level elements have a `Next` slot; 121 | lower-level elements do not. 122 | 123 | ## `StateMachineStateIR` 124 | 125 | Has a name, a collection of key/value pairs, and an optional 'next 126 | state name'. Names are assigned incrementally via a class attribute. 127 | The 'value' is accessible via `value_as_json_obj()`. 128 | 129 | ## `StateMachineFragmentIR` 130 | 131 | A fragment of a state machine, representing some well-defined piece of 132 | the source program. For example, an `if` statement. Has a collection 133 | of states, knowledge of which state is the 'entry' state, and 134 | knowledge of which states, if any, are 'exit' states. 135 | 136 | The whole fragment can be connected to its correct 'next' state via 137 | the `set_next_state()` method. 138 | 139 | Can be turned into a JSON-friendly object via `as_json_obj()`. 140 | Components are stored as possibly nested JSON-friendly objects. 141 | 142 | The following objects can be converted into fragments, via an 143 | `as_fragment()` method: 144 | 145 | ### `FunctionCallIR` 146 | 147 | As a state-machine fragment, consists of a bit of a dance to pass the 148 | details of which function, called with which arguments, to the Lambda 149 | function. Uses a `Pass` state to inject a 'call descriptor' into the 150 | state, and then a `Task` state to perform the call and inject the 151 | results into the appropriate slot within `locals`. 152 | 153 | ### `ParallelIR` 154 | 155 | Because each branch of a `Parallel` state is its own self-contained 156 | state machine, we find that state-machine and then 'render' it into 157 | its JSON-friendly form. All such forms are gathered together into the 158 | resulting `Parallel` state representation, which is then the sole 159 | state of the resulting fragment. 160 | 161 | ### `RaiseIR` 162 | 163 | State-machine fragment is just one `Fail` state. 164 | 165 | ### `ReturnIR` 166 | 167 | State-machine fragment is just one `Succeed` state, pulling out the 168 | appropriate variable from `$.locals` via its `InputPath`. 169 | 170 | ### `AssignmentIR` 171 | 172 | An assignment is from either a simple function call (with optional 173 | retry-specs), or from a `Parallel` call. The source of the assignment 174 | knows how to construct the fragment, so the `AssignmentIR` delegates 175 | to its `source`. 176 | 177 | The source can be either a `FunctionCallIR` or a `ParallelIR`. 178 | 179 | ### `SuiteIR` 180 | 181 | A simple chain of fragments, each one having its next state set to the 182 | 'enter state' of the following fragment. 183 | 184 | ### `IfIR` 185 | 186 | A `Choice` state with only one choice clause, corresponding to the 187 | `True` branch of the Python-level `if`. The `else` clause becomes the 188 | `Default` state. 189 | 190 | ### `TryIR` 191 | 192 | The state-machine semantics are such that only a single `Task` can 193 | have `Catch` clauses, so at the Python level, the body of a `try` must 194 | consist of exactly one assignment. As a state machine fragment, we 195 | create the assignment's fragment, then punch in a `Catch` field 196 | linking to all the fragments arising from the Python-level `except` 197 | clauses. 198 | 199 | 200 | # Lambda code 201 | 202 | For invoking functions via the Lambda machinery, we create a single 203 | 'wrapper'/'dispatcher' Lambda. This has a very short entry-point 204 | function, which finds the required function name and arg names from 205 | the input dict. It then finds the required function in the original 206 | module, looks up the arguments from `locals`, and returns the result 207 | of calling that function on those args. This is all created by the 208 | 'wrapper-compiler', `pysfnwc.py`. 209 | 210 | 211 | --- 212 | 213 | This document: Copyright 2018 Ben North; licensed under 214 | [CC BY-SA 4.0](http://creativecommons.org/licenses/by-sa/4.0/) 215 | 216 | See the file `COPYING` for full licensing details. 217 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Install script for ``pysfn``. 2 | 3 | Usage:: 4 | 5 | pip install pysfn 6 | """ 7 | 8 | import pathlib 9 | import setuptools 10 | 11 | parent = pathlib.Path(__file__).parent 12 | long_description = (parent / "README.md").read_text() 13 | version = (parent / "VERSION").read_text().strip() 14 | classifiers = [ 15 | "Environment :: Console", 16 | "Intended Audience :: Developers", 17 | "Programming Language :: Python :: 3 :: Only", 18 | "Programming Language :: Python :: 3.6", 19 | "Programming Language :: Python :: 3.7", 20 | "Programming Language :: Python :: 3.8", 21 | "Natural Language :: English", 22 | "Operating System :: OS Independent", 23 | ( 24 | "License :: OSI Approved :: GNU General Public License v3 or later " 25 | "(GPLv3+)")] 26 | 27 | setuptools.setup( 28 | name="pysfn", 29 | version=version, 30 | license="GPL3", 31 | author="Ben North", 32 | author_email="ben@redfrontdoor.org", 33 | maintainer="Ben North", 34 | maintainer_email="ben@redfrontdoor.org", 35 | description="Compile Python code to AWS Step Function", 36 | long_description=long_description, 37 | long_description_content_type="text/markdown", 38 | url="https://github.com/bennorth/pyawssfn", 39 | classifiers=classifiers, 40 | keywords="aws sfn lambda step functions compiler", 41 | packages=setuptools.find_packages(where="src"), 42 | package_dir={"": "src"}, 43 | python_requires="~=3.6", 44 | install_requires=["click", "attrs"], 45 | extras_require={"dev": ["pytest"]}, 46 | project_urls={"Bugs": "https://github.com/bennorth/pyawssfn/issues"}) 47 | -------------------------------------------------------------------------------- /src/pysfn/__init__.py: -------------------------------------------------------------------------------- 1 | """Python to AWS Step Functions compiler.""" 2 | 3 | from .definition import * 4 | -------------------------------------------------------------------------------- /src/pysfn/definition.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Ben North 2 | # 3 | # This file is part of 'plausibility argument of concept for compiling 4 | # Python into Amazon Step Function state machine JSON'. 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | 20 | def StringEquals(x, y): 21 | assert isinstance(x, str) 22 | assert isinstance(y, str) 23 | return x == y 24 | 25 | 26 | class Fail(Exception): 27 | def __init__(self, label, message): 28 | self.label = label 29 | self.message = message 30 | 31 | def __str__(self): 32 | return f'{self.label}: {self.message}' 33 | 34 | 35 | def parallel(*funs): 36 | return [f() for f in funs] 37 | 38 | 39 | def with_retry_spec(fun, args, *retry_specs): 40 | return fun(*args) 41 | 42 | 43 | def main(fun): 44 | return fun 45 | -------------------------------------------------------------------------------- /src/pysfn/tools/__init__.py: -------------------------------------------------------------------------------- 1 | """Tools for ``pysfn``.""" 2 | -------------------------------------------------------------------------------- /src/pysfn/tools/compile.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Ben North 2 | # 3 | # This file is part of 'plausibility argument of concept for compiling 4 | # Python into Amazon Step Function state machine JSON'. 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | 20 | import ast 21 | import attr 22 | from functools import reduce 23 | from operator import concat 24 | import click 25 | import json 26 | 27 | 28 | ######################################################################## 29 | 30 | def psf_attr(nd, raise_if_not=True): 31 | """ 32 | Extract the attribute name from an AST node of the form 33 | 34 | PSF.something 35 | 36 | If the given AST node is not of that form, either raise a 37 | ValueError (if raise_if_not is True), or return None (if 38 | raise_if_not is False). 39 | """ 40 | attr_val = None 41 | if isinstance(nd, ast.Attribute): 42 | val = nd.value 43 | if isinstance(val, ast.Name) and val.id == 'PSF': 44 | attr_val = nd.attr 45 | if attr_val is None and raise_if_not: 46 | raise ValueError('expected PSF.something') 47 | return attr_val 48 | 49 | 50 | def chained_key(nd): 51 | """ 52 | Given an AST node representing a value like 53 | 54 | foo['bar']['baz'] 55 | 56 | return a list of the components involved; here, 57 | 58 | ['foo', 'bar', 'baz'] 59 | 60 | If the given node is not of that form, raise a ValueError. 61 | """ 62 | if isinstance(nd, ast.Name): 63 | return [nd.id] 64 | if isinstance(nd, ast.Subscript): 65 | if isinstance(nd.slice, ast.Index): 66 | if isinstance(nd.slice.value, ast.Str): 67 | suffix = nd.slice.value.s 68 | if isinstance(nd.value, ast.Name): 69 | prefix = [nd.value.id] 70 | else: 71 | prefix = chained_key(nd.value) 72 | return prefix + [suffix] 73 | raise ValueError('expected chained lookup via strings on name') 74 | 75 | 76 | def chained_key_smr(k): 77 | """ 78 | Convert a sequence of chained lookups into the jsonPath which will 79 | refer to its location in the 'locals' object. 80 | """ 81 | return '.'.join(['$', 'locals'] + k) 82 | 83 | 84 | def lmap(f, xs): 85 | return list(map(f, xs)) 86 | 87 | 88 | def maybe_with_next(base_fields, next_state_name): 89 | """ 90 | Return a copy of base_fields (a dict), with an additional item 91 | 92 | 'Next': next_state_name 93 | 94 | iff next_state_name is non-None. 95 | """ 96 | obj = dict(base_fields) 97 | if next_state_name is not None: 98 | obj['Next'] = next_state_name 99 | return obj 100 | 101 | 102 | ######################################################################## 103 | 104 | class ChoiceConditionIR: 105 | @staticmethod 106 | def from_ast_node(nd): 107 | if isinstance(nd, ast.Call): 108 | return TestComparisonIR.from_ast_node(nd) 109 | elif isinstance(nd, ast.BoolOp): 110 | return TestCombinatorIR.from_ast_node(nd) 111 | raise ValueError('expected Call') 112 | 113 | 114 | @attr.s 115 | class TestComparisonIR(ChoiceConditionIR): 116 | predicate_name = attr.ib() 117 | predicate_variable = attr.ib() 118 | predicate_literal = attr.ib() 119 | 120 | @classmethod 121 | def from_ast_node(cls, nd): 122 | if isinstance(nd, ast.Call) and len(nd.args) == 2: 123 | return cls(psf_attr(nd.func), 124 | chained_key(nd.args[0]), 125 | nd.args[1].s) 126 | raise ValueError('expected function-call PSF.something(...)') 127 | 128 | def as_choice_rule_smr(self, next_state_name): 129 | return maybe_with_next( 130 | {'Variable': chained_key_smr(self.predicate_variable), 131 | self.predicate_name: self.predicate_literal}, 132 | next_state_name) 133 | 134 | 135 | @attr.s 136 | class TestCombinatorIR(ChoiceConditionIR): 137 | opname = attr.ib() 138 | values = attr.ib() 139 | 140 | @classmethod 141 | def from_ast_node(cls, nd): 142 | if isinstance(nd, ast.BoolOp): 143 | if isinstance(nd.op, ast.Or): 144 | opname = 'Or' 145 | elif isinstance(nd.op, ast.And): 146 | opname = 'And' 147 | else: 148 | raise ValueError('expected Or or And') 149 | return cls(opname, lmap(ChoiceConditionIR.from_ast_node, nd.values)) 150 | raise ValueError('expected BoolOp') 151 | 152 | def as_choice_rule_smr(self, next_state_name): 153 | terms = [v.as_choice_rule_smr(None) for v in self.values] 154 | return maybe_with_next( 155 | {self.opname: terms}, 156 | next_state_name) 157 | 158 | 159 | ######################################################################## 160 | 161 | @attr.s 162 | class RetrySpecIR: 163 | error_equals = attr.ib() 164 | interval_seconds = attr.ib() 165 | max_attempts = attr.ib() 166 | backoff_rate = attr.ib() 167 | 168 | @classmethod 169 | def from_ast_node(cls, nd): 170 | return cls([error_name.s for error_name in nd.elts[0].elts], 171 | nd.elts[1].n, 172 | nd.elts[2].n, 173 | nd.elts[3].n) 174 | 175 | def as_json_obj(self): 176 | return {'ErrorEquals': self.error_equals, 177 | 'IntervalSeconds': self.interval_seconds, 178 | 'MaxAttempts': self.max_attempts, 179 | 'BackoffRate': self.backoff_rate} 180 | 181 | 182 | @attr.s 183 | class CatcherIR: 184 | error_equals = attr.ib() 185 | body = attr.ib() 186 | 187 | @classmethod 188 | def from_ast_node(cls, nd): 189 | return cls([nd.type.id], SuiteIR.from_ast_nodes(nd.body)) 190 | 191 | 192 | ######################################################################## 193 | 194 | @attr.s 195 | class ReturnIR: 196 | varname = attr.ib() 197 | 198 | @classmethod 199 | def from_ast_node(cls, nd): 200 | if isinstance(nd.value, ast.Name): 201 | return cls(nd.value.id) 202 | raise ValueError('expected return of variable') 203 | 204 | def as_fragment(self, xln_ctx): 205 | s = StateMachineStateIR.from_fields( 206 | Type='Succeed', 207 | InputPath=chained_key_smr([self.varname])) 208 | return StateMachineFragmentIR([s], s, []) 209 | 210 | 211 | @attr.s 212 | class RaiseIR: 213 | error = attr.ib() 214 | cause = attr.ib() 215 | 216 | @classmethod 217 | def from_ast_node(cls, nd): 218 | if (isinstance(nd.exc, ast.Call) 219 | and psf_attr(nd.exc.func) == 'Fail' 220 | and len(nd.exc.args) == 2 221 | and isinstance(nd.exc.args[0], ast.Str) 222 | and isinstance(nd.exc.args[1], ast.Str)): 223 | return cls(nd.exc.args[0].s, nd.exc.args[1].s) 224 | raise ValueError('expected raise PSF.Fail("foo", "bar")') 225 | 226 | def as_fragment(self, xln_ctx): 227 | s = StateMachineStateIR.from_fields( 228 | Type='Fail', Error=self.error, Cause=self.cause) 229 | return StateMachineFragmentIR([s], s, []) 230 | 231 | 232 | class AssignmentSourceIR: 233 | @classmethod 234 | def from_ast_node(cls, nd, defs): 235 | if isinstance(nd, ast.Call): 236 | if (isinstance(nd.func, ast.Name) 237 | or (isinstance(nd.func, ast.Attribute) 238 | and psf_attr(nd.func) == 'with_retry_spec')): 239 | return FunctionCallIR.from_ast_node(nd) 240 | if (isinstance(nd.func, ast.Attribute) 241 | and psf_attr(nd.func) == 'parallel'): 242 | return ParallelIR.from_ast_node_and_defs(nd, defs) 243 | raise ValueError('expected fn(x, y)' 244 | ' or PSF.with_retry_spec(fn, (x, y), s1, s2)') 245 | 246 | 247 | @attr.s 248 | class FunctionCallIR(AssignmentSourceIR): 249 | fun_name = attr.ib() 250 | arg_names = attr.ib() 251 | retry_spec = attr.ib() 252 | 253 | @classmethod 254 | def from_ast_node(cls, nd): 255 | if isinstance(nd, ast.Call): 256 | if not isinstance(nd.func, ast.Attribute): 257 | # Bare call 258 | return cls(nd.func.id, [a.id for a in nd.args], None) 259 | elif psf_attr(nd.func) == 'with_retry_spec': 260 | return cls(nd.args[0].id, 261 | [a.id for a in nd.args[1].elts], 262 | lmap(RetrySpecIR.from_ast_node, nd.args[2:])) 263 | raise ValueError('expected some_function(some, args)' 264 | ' or PSF.with_retry_spec(fun, (some, args),' 265 | ' retry_spec_1, retry_spec_2)') 266 | 267 | def call_descriptor(self): 268 | return {"function": self.fun_name, "arg_names": self.arg_names} 269 | 270 | def as_fragment(self, xln_ctx, target_varname): 271 | s_pass = StateMachineStateIR.from_fields(Type='Pass', 272 | Result=self.call_descriptor(), 273 | ResultPath='$.call_descr') 274 | 275 | task_fields = {'Type': 'Task', 276 | 'Resource': xln_ctx.lambda_arn, 277 | 'ResultPath': chained_key_smr([target_varname])} 278 | if self.retry_spec is not None: 279 | task_fields['Retry'] = [s.as_json_obj() for s in self.retry_spec] 280 | s_task = StateMachineStateIR.from_fields(**task_fields) 281 | 282 | s_pass.next_state_name = s_task.name 283 | 284 | return StateMachineFragmentIR([s_pass, s_task], s_pass, [s_task]) 285 | 286 | 287 | @attr.s 288 | class ParallelIR: 289 | branches = attr.ib() 290 | 291 | @classmethod 292 | def from_ast_node_and_defs(cls, nd, defs): 293 | branch_names = [arg.id for arg in nd.args] 294 | return cls([defs[n] for n in branch_names]) 295 | 296 | def as_fragment(self, xln_ctx, target_varname): 297 | # The branches of a 'Parallel' are isolated state machines, so 298 | # we need to convert each one into a JSON-friendly form now. 299 | # This is in contrast to 'If' or 'Try' where the bodies 300 | # contribute their states to the top-level state machine. 301 | s_parallel = StateMachineStateIR.from_fields( 302 | Type='Parallel', 303 | Branches=[branch.as_fragment(xln_ctx).as_json_obj() 304 | for branch in self.branches], 305 | ResultPath=chained_key_smr([target_varname])) 306 | return StateMachineFragmentIR([s_parallel], s_parallel, [s_parallel]) 307 | 308 | 309 | class StatementIR: 310 | @classmethod 311 | def from_ast_node(self, nd, defs): 312 | if isinstance(nd, ast.Assign): 313 | return AssignmentIR.from_ast_node(nd, defs) 314 | if isinstance(nd, ast.Try): 315 | return TryIR.from_ast_node(nd) 316 | if isinstance(nd, ast.If): 317 | return IfIR.from_ast_node(nd) 318 | if isinstance(nd, ast.Return): 319 | return ReturnIR.from_ast_node(nd) 320 | if isinstance(nd, ast.Raise): 321 | return RaiseIR.from_ast_node(nd) 322 | raise ValueError('unexpected node type {} for statement' 323 | .format(type(nd))) 324 | 325 | 326 | @attr.s 327 | class AssignmentIR(StatementIR): 328 | target_varname = attr.ib() 329 | source = attr.ib() 330 | 331 | @classmethod 332 | def from_ast_node(cls, nd, defs): 333 | if isinstance(nd, ast.Assign) and len(nd.targets) == 1: 334 | return cls(nd.targets[0].id, 335 | AssignmentSourceIR.from_ast_node(nd.value, defs)) 336 | raise ValueError('expected single-target assignment') 337 | 338 | def as_fragment(self, xln_ctx): 339 | return self.source.as_fragment(xln_ctx, self.target_varname) 340 | 341 | 342 | @attr.s 343 | class TryIR(StatementIR): 344 | body = attr.ib() 345 | catchers = attr.ib() 346 | 347 | @classmethod 348 | def from_ast_node(cls, nd): 349 | assert len(nd.body) == 1 350 | body = SuiteIR.from_ast_nodes(nd.body) 351 | return cls(body, [CatcherIR.from_ast_node(h) for h in nd.handlers]) 352 | 353 | def as_fragment(self, xln_ctx): 354 | body = self.body.as_fragment(xln_ctx) 355 | catcher_fragments = [c.body.as_fragment(xln_ctx) for c in self.catchers] 356 | s_task = body.all_states[1] 357 | assert s_task.fields['Type'] == 'Task' 358 | s_task.fields['Catch'] = [ 359 | {'ErrorEquals': c.error_equals, 'Next': f.enter_state.name} 360 | for (c, f) in zip(self.catchers, catcher_fragments)] 361 | 362 | all_catcher_states = reduce( 363 | concat, [f.all_states for f in catcher_fragments], []) 364 | 365 | all_catcher_exits = reduce( 366 | concat, [f.exit_states for f in catcher_fragments], []) 367 | 368 | return StateMachineFragmentIR( 369 | body.all_states + all_catcher_states, 370 | body.enter_state, 371 | body.exit_states + all_catcher_exits) 372 | 373 | 374 | @attr.s 375 | class IfIR(StatementIR): 376 | test = attr.ib() 377 | true_body = attr.ib() 378 | false_body = attr.ib() 379 | 380 | @classmethod 381 | def from_ast_node(cls, nd): 382 | return cls(ChoiceConditionIR.from_ast_node(nd.test), 383 | SuiteIR.from_ast_nodes(nd.body), 384 | SuiteIR.from_ast_nodes(nd.orelse)) 385 | 386 | def as_fragment(self, xln_ctx): 387 | true_frag = self.true_body.as_fragment(xln_ctx) 388 | false_frag = self.false_body.as_fragment(xln_ctx) 389 | 390 | choice_rule = self.test.as_choice_rule_smr(true_frag.enter_state.name) 391 | choice_state = StateMachineStateIR.from_fields( 392 | Type='Choice', 393 | Choices=[choice_rule], 394 | Default=false_frag.enter_state.name) 395 | 396 | all_states = ([choice_state] 397 | + true_frag.all_states 398 | + false_frag.all_states) 399 | 400 | exit_states = true_frag.exit_states + false_frag.exit_states 401 | 402 | return StateMachineFragmentIR(all_states, choice_state, exit_states) 403 | 404 | 405 | @attr.s 406 | class SuiteIR: 407 | body = attr.ib() 408 | 409 | @classmethod 410 | def from_ast_nodes(cls, nds): 411 | body = [] 412 | defs = {} 413 | for nd in nds: 414 | if isinstance(nd, ast.FunctionDef): 415 | defs[nd.name] = SuiteIR.from_ast_nodes(nd.body) 416 | else: 417 | body.append(StatementIR.from_ast_node(nd, defs)) 418 | return cls(body) 419 | 420 | def as_fragment(self, xln_ctx): 421 | fragments = [stmt.as_fragment(xln_ctx) for stmt in self.body] 422 | for f0, f1 in zip(fragments[:-1], fragments[1:]): 423 | f0.set_next_state(f1.enter_state.name) 424 | return StateMachineFragmentIR( 425 | reduce(concat, [f.all_states for f in fragments], []), 426 | fragments[0].enter_state, 427 | fragments[-1].exit_states) 428 | 429 | 430 | ######################################################################## 431 | 432 | @attr.s 433 | class TranslationContext: 434 | lambda_arn = attr.ib() 435 | 436 | @staticmethod 437 | def is_main_fundef(fd): 438 | return ( 439 | isinstance(fd, ast.FunctionDef) 440 | and len(fd.decorator_list) == 1 441 | and psf_attr(fd.decorator_list[0], raise_if_not=False) == 'main') 442 | 443 | def state_machine_main_fundef(self, syntax_tree): 444 | candidates = [x for x in syntax_tree.body if self.is_main_fundef(x)] 445 | if len(candidates) != 1: 446 | raise ValueError('no unique PSF.main function') 447 | return candidates[0] 448 | 449 | def top_level_state_machine(self, syntax_tree): 450 | fun = self.state_machine_main_fundef(syntax_tree) 451 | suite = SuiteIR.from_ast_nodes(fun.body) 452 | return suite.as_fragment(self) 453 | 454 | 455 | @attr.s 456 | class StateMachineStateIR: 457 | name = attr.ib() 458 | fields = attr.ib() 459 | next_state_name = attr.ib() 460 | 461 | next_id = 0 462 | 463 | @classmethod 464 | def from_fields(cls, **kwargs): 465 | name = 'n{}'.format(cls.next_id) 466 | cls.next_id += 1 467 | return cls(name, kwargs, None) 468 | 469 | def value_as_json_obj(self): 470 | return maybe_with_next(self.fields, self.next_state_name) 471 | 472 | 473 | @attr.s 474 | class StateMachineFragmentIR: 475 | all_states = attr.ib() 476 | enter_state = attr.ib() 477 | exit_states = attr.ib() 478 | 479 | @property 480 | def n_states(self): 481 | return len(self.all_states) 482 | 483 | def set_next_state(self, next_state_name): 484 | for s in self.exit_states: 485 | s.next_state_name = next_state_name 486 | 487 | def as_json_obj(self): 488 | return {'States': {s.name: s.value_as_json_obj() 489 | for s in self.all_states}, 490 | 'StartAt': self.enter_state.name} 491 | 492 | 493 | ######################################################################## 494 | 495 | @click.command() 496 | @click.argument('source_fname') 497 | @click.argument('lambda_arn') 498 | def main(source_fname, lambda_arn): 499 | syntax_tree = ast.parse(source=open(source_fname, 'rt').read(), 500 | filename=source_fname) 501 | 502 | xln_ctx = TranslationContext(lambda_arn) 503 | state_machine = xln_ctx.top_level_state_machine(syntax_tree) 504 | print(json.dumps(state_machine.as_json_obj(), indent=2)) 505 | 506 | 507 | if __name__ == '__main__': 508 | main() 509 | -------------------------------------------------------------------------------- /src/pysfn/tools/gen_lambda.py: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2018 Ben North 2 | # 3 | # This file is part of 'plausibility argument of concept for compiling 4 | # Python into Amazon Step Function state machine JSON'. 5 | # 6 | # This program is free software: you can redistribute it and/or modify 7 | # it under the terms of the GNU General Public License as published by 8 | # the Free Software Foundation, either version 3 of the License, or 9 | # (at your option) any later version. 10 | # 11 | # This program is distributed in the hope that it will be useful, 12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | # GNU General Public License for more details. 15 | # 16 | # You should have received a copy of the GNU General Public License 17 | # along with this program. If not, see . 18 | 19 | 20 | import click 21 | import zipfile 22 | import os 23 | import os.path 24 | from contextlib import closing 25 | 26 | 27 | package_dir = os.path.split(os.path.split(__file__)[0])[0] 28 | template = """\ 29 | import sys 30 | sys.path.insert(0, './inner') 31 | 32 | import inner.{code_modulename} as inner_module 33 | 34 | def dispatch(event, context): 35 | fun = getattr(inner_module, event['call_descr']['function']) 36 | args = [event['locals'][arg_name] 37 | for arg_name in event['call_descr']['arg_names']] 38 | return fun(*args) 39 | """ 40 | 41 | 42 | def zinfo(fname): 43 | # https://stackoverflow.com/questions/46076543 44 | zi = zipfile.ZipInfo(fname) 45 | zi.external_attr = 0o777 << 16 46 | return zi 47 | 48 | 49 | @click.command() 50 | @click.argument('code_filename') 51 | @click.argument('zip_filename') 52 | def compile_zipfile(code_filename, zip_filename): 53 | code_basename = os.path.basename(code_filename) 54 | code_modulename = os.path.splitext(code_basename)[0] 55 | handler_content = template.format(code_modulename=code_modulename) 56 | with closing(zipfile.ZipFile(zip_filename, 'x')) as f_zip: 57 | f_zip.writestr(zinfo('handler.py'), handler_content) 58 | f_zip.write(code_filename, 'inner/{}'.format(code_basename)) 59 | f_zip.write(os.path.join(package_dir, 'definition.py'), 'pysfn.py') 60 | 61 | 62 | if __name__ == '__main__': 63 | compile_zipfile() 64 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """Tests for ``pysfn``.""" 2 | -------------------------------------------------------------------------------- /tests/test_analyse_text.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from examples import analyse_text as A 3 | from pysfn import definition as PSF 4 | 5 | 6 | class TestAnalysis: 7 | def test_get_summary(self): 8 | assert A.get_summary('hello world') == {'head': 'h'} 9 | 10 | def test_get_summary_too_short(self): 11 | with pytest.raises(A.TextTooShortError): 12 | A.get_summary('') 13 | 14 | def test_augment_summary(self): 15 | summary = {'head': 'h'} 16 | aug_summary = A.augment_summary('hello world', summary) 17 | assert aug_summary['head'] == 'h' # Original data should remain 18 | assert aug_summary['n_characters'] == 11 19 | 20 | def test_get_n_vowels(self): 21 | assert A.get_n_vowels('hello world') == 3 22 | assert A.get_n_vowels('rhythms') == 0 23 | 24 | def test_get_n_spaces(self): 25 | assert A.get_n_spaces('hello world') == 1 26 | assert A.get_n_spaces('goodbye') == 0 27 | assert A.get_n_spaces('once upon a time') == 3 28 | 29 | def test_format_result(self): 30 | summary = {'head': 'h', 'n_characters': 10} 31 | infos = [42, 99] 32 | assert (A.format_result(summary, infos) 33 | == ('text starts with h,' 34 | ' has 10 chars,' 35 | ' 42 vowels, and 99 spaces')) 36 | 37 | def test_summary(self): 38 | got = A.summarise('a short example') 39 | assert got == ('text starts with a, has 15 chars,' 40 | ' 5 vowels, and 2 spaces') 41 | 42 | def test_c_summary(self): 43 | got = A.summarise('choose wisely') 44 | assert got == 'text starts with "c"; look: "c"' 45 | 46 | def test_summary_too_short(self): 47 | with pytest.raises(PSF.Fail, match='text too short'): 48 | A.summarise('') 49 | 50 | def test_summary_wrong_start(self): 51 | with pytest.raises(PSF.Fail, match='wrong starting letter'): 52 | A.summarise('do not handle starting with "d"') 53 | -------------------------------------------------------------------------------- /tests/test_pysfnc.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | from pysfn.tools import compile as C 3 | import ast 4 | import textwrap 5 | from functools import partial 6 | 7 | 8 | def stmt_value(txt): 9 | return ast.parse(textwrap.dedent(txt)).body[0] 10 | 11 | 12 | def expr_value(txt): 13 | return stmt_value(txt).value 14 | 15 | 16 | def suite_value(txt): 17 | return ast.parse(textwrap.dedent(txt)).body 18 | 19 | 20 | def find_state_by_name(frag, name): 21 | candidates = [s for s in frag.all_states if s.name == name] 22 | assert len(candidates) == 1 23 | return candidates[0] 24 | 25 | 26 | def find_successor_state(frag, state): 27 | return find_state_by_name(frag, state.next_state_name) 28 | 29 | 30 | def _test_factory_raises(nd, factory): 31 | with pytest.raises(ValueError): 32 | factory(nd) 33 | 34 | 35 | def _assert_is_assignment(ir, target, src_funname, *src_argnames): 36 | assert ir.target_varname == target 37 | assert ir.source.fun_name == src_funname 38 | assert ir.source.arg_names == list(src_argnames) 39 | 40 | 41 | def _assert_is_return(ir, exp_var_name): 42 | assert isinstance(ir, C.ReturnIR) 43 | assert ir.varname == exp_var_name 44 | 45 | 46 | def _assert_comparison_correct(cmp, exp_name, exp_variable, exp_literal): 47 | assert cmp.predicate_name == exp_name 48 | assert cmp.predicate_variable == exp_variable 49 | assert cmp.predicate_literal == exp_literal 50 | 51 | 52 | def _assert_state_pair_forms_assignment(s0, s1, xln_ctx, 53 | assign_target, fun, args): 54 | assert s0.fields == {'Type': 'Pass', 55 | 'Result': {'function': fun, 'arg_names': args}, 56 | 'ResultPath': '$.call_descr'} 57 | assert s0.next_state_name == s1.name 58 | assert s1.fields['Type'] == 'Task' 59 | assert s1.fields['Resource'] == xln_ctx.lambda_arn 60 | assert s1.fields['ResultPath'] == f'$.locals.{assign_target}' 61 | # Ignore 'Retry' content, if any. 62 | 63 | 64 | mk_statement_empty_defs = partial(C.StatementIR.from_ast_node, defs={}) 65 | mk_assign_src_empty_defs = partial(C.AssignmentSourceIR.from_ast_node, defs={}) 66 | 67 | 68 | @pytest.fixture(scope='module') 69 | def translation_context(): 70 | return C.TranslationContext('arn:...:function:dispatch') 71 | 72 | 73 | class TestSupportFunctions: 74 | def test_psf_attr(self): 75 | val = expr_value('PSF.hello_world') 76 | assert C.psf_attr(val) == 'hello_world' 77 | 78 | @pytest.mark.parametrize( 79 | 'text', 80 | ['99 + 42', 81 | 'something_else.odd', 82 | 'PSF.nested.attribute'] 83 | ) 84 | def test_psf_attr_bad_input(self, text): 85 | val = expr_value(text) 86 | with pytest.raises(ValueError): 87 | C.psf_attr(val) 88 | assert C.psf_attr(val, raise_if_not=False) is None 89 | 90 | def test_chained_key(self): 91 | val = expr_value('foo["bar"]["baz"]') 92 | assert C.chained_key(val) == ['foo', 'bar', 'baz'] 93 | 94 | def test_simple_chained_key(self): 95 | val = expr_value('foo') 96 | assert C.chained_key(val) == ['foo'] 97 | 98 | @pytest.mark.parametrize( 99 | 'text', 100 | ['1 + 1', 101 | 'some_dict[3]["foo"]', 102 | 'some_obj[slice_lb:slice_ub]', 103 | 'some_obj.attrib_access'] 104 | ) 105 | def test_chained_key_bad_input(self, text): 106 | val = expr_value(text) 107 | with pytest.raises(ValueError): 108 | C.chained_key(val) 109 | 110 | def test_chained_key_smr(self): 111 | assert C.chained_key_smr(['foo']) == '$.locals.foo' 112 | assert C.chained_key_smr(['foo', 'bar']) == '$.locals.foo.bar' 113 | 114 | def test_maybe_with_next(self): 115 | assert (C.maybe_with_next({'foo': 99}, None) 116 | == {'foo': 99}) 117 | assert (C.maybe_with_next({'foo': 99}, 'done') 118 | == {'foo': 99, 'Next': 'done'}) 119 | 120 | 121 | class TestChoice: 122 | @pytest.fixture(scope='module', 123 | params=[C.TestComparisonIR, C.ChoiceConditionIR]) 124 | def cmp_class(self, request): 125 | return request.param 126 | 127 | @pytest.fixture(scope='module', 128 | params=[C.TestCombinatorIR, C.ChoiceConditionIR]) 129 | def comb_class(self, request): 130 | return request.param 131 | 132 | @staticmethod 133 | def _test_comparison(cmp_class, text, name, variable, literal): 134 | val = expr_value(text) 135 | cmp = cmp_class.from_ast_node(val) 136 | _assert_comparison_correct(cmp, name, variable, literal) 137 | 138 | def test_comparison(self, cmp_class): 139 | self._test_comparison(cmp_class, 140 | 'PSF.StringEquals(foo, "bar")', 141 | 'StringEquals', ['foo'], 'bar') 142 | 143 | def test_chained_comparison(self, cmp_class): 144 | self._test_comparison(cmp_class, 145 | 'PSF.StringEquals(foo["bar"], "baz")', 146 | 'StringEquals', ['foo', 'bar'], 'baz') 147 | 148 | @pytest.mark.parametrize( 149 | 'text', 150 | ['1 == 1', 'random_check(a, b)'] 151 | ) 152 | def test_comparison_bad_input(self, cmp_class, text): 153 | _test_factory_raises(expr_value(text), cmp_class.from_ast_node) 154 | 155 | @pytest.mark.parametrize( 156 | 'op, exp_opname', 157 | [('or', 'Or'), ('and', 'And')] 158 | ) 159 | def test_combinator(self, comb_class, op, exp_opname): 160 | val = expr_value(f'PSF.StringEquals(foo, "x")' 161 | f' {op} PSF.StringEquals(foo["bar"], "y")') 162 | choice = comb_class.from_ast_node(val) 163 | assert choice.opname == exp_opname 164 | _assert_comparison_correct(choice.values[0], 165 | 'StringEquals', ['foo'], 'x') 166 | _assert_comparison_correct(choice.values[1], 167 | 'StringEquals', ['foo', 'bar'], 'y') 168 | 169 | @pytest.mark.parametrize( 170 | 'text', 171 | ['1 == 1', 'random_check(a, b)', 'x < 77'] 172 | ) 173 | def test_combinator_bad_input(self, comb_class, text): 174 | _test_factory_raises(expr_value(text), comb_class.from_ast_node) 175 | 176 | def test_comparison_conversion_to_smr(self): 177 | val = expr_value('PSF.StringEquals(foo, "x")') 178 | choice = C.ChoiceConditionIR.from_ast_node(val) 179 | smr = choice.as_choice_rule_smr('wash_dishes') 180 | assert smr == {'Variable': '$.locals.foo', 181 | 'StringEquals': 'x', 182 | 'Next': 'wash_dishes'} 183 | 184 | def test_combinator_conversion_to_smr(self): 185 | val = expr_value('PSF.StringEquals(foo, "x")' 186 | ' or PSF.StringEquals(foo["bar"], "y")') 187 | choice = C.ChoiceConditionIR.from_ast_node(val) 188 | smr = choice.as_choice_rule_smr('wash_dishes') 189 | assert smr == {'Or': [{'Variable': '$.locals.foo', 190 | 'StringEquals': 'x'}, 191 | {'Variable': '$.locals.foo.bar', 192 | 'StringEquals': 'y'}], 193 | 'Next': 'wash_dishes'} 194 | 195 | 196 | class TestRetrySpec: 197 | def test_retry_spec(self): 198 | expr = expr_value('(["BadThing", "WorseThing"], 2.5, 3, 2.0)') 199 | ir = C.RetrySpecIR.from_ast_node(expr) 200 | assert ir.error_equals == ['BadThing', 'WorseThing'] 201 | assert ir.interval_seconds == 2.5 202 | assert ir.max_attempts == 3 203 | assert ir.backoff_rate == 2.0 204 | 205 | def test_as_json(self): 206 | expr = expr_value('(["BadThing", "WorseThing"], 2.5, 3, 2.0)') 207 | ir = C.RetrySpecIR.from_ast_node(expr) 208 | obj = ir.as_json_obj() 209 | assert obj == {'ErrorEquals': ['BadThing', 'WorseThing'], 210 | 'IntervalSeconds': 2.5, 211 | 'MaxAttempts': 3, 212 | 'BackoffRate': 2.0} 213 | 214 | 215 | @pytest.fixture 216 | def sample_try_stmt(scope='module'): 217 | return stmt_value(""" 218 | try: 219 | x = f(y) 220 | except BadThing: 221 | foo = bar(baz) 222 | qux = hello(world) 223 | except WorseThing: 224 | qux = bar(baz) 225 | foo = hello(world) 226 | """) 227 | 228 | 229 | def _assert_sample_try_catchers_correct(catchers): 230 | assert len(catchers) == 2 231 | assert catchers[0].error_equals == ['BadThing'] 232 | _assert_is_assignment(catchers[0].body.body[0], 'foo', 'bar', 'baz') 233 | _assert_is_assignment(catchers[0].body.body[1], 'qux', 'hello', 'world') 234 | assert catchers[1].error_equals == ['WorseThing'] 235 | _assert_is_assignment(catchers[1].body.body[0], 'qux', 'bar', 'baz') 236 | _assert_is_assignment(catchers[1].body.body[1], 'foo', 'hello', 'world') 237 | 238 | 239 | class TestCatcher: 240 | def test_catcher(self, sample_try_stmt): 241 | handlers = sample_try_stmt.handlers 242 | catchers = [C.CatcherIR.from_ast_node(h) for h in handlers] 243 | _assert_sample_try_catchers_correct(catchers) 244 | 245 | 246 | class TestReturnIR: 247 | @pytest.fixture(scope='module', 248 | params=[C.ReturnIR.from_ast_node, mk_statement_empty_defs]) 249 | def factory(self, request): 250 | return request.param 251 | 252 | def test_return(self, factory): 253 | stmt = stmt_value('return banana') 254 | ir = factory(stmt) 255 | _assert_is_return(ir, 'banana') 256 | 257 | def test_return_bad_input(self, factory): 258 | _test_factory_raises(stmt_value('return 42'), factory) 259 | 260 | def test_as_fragment(self, translation_context): 261 | stmt = stmt_value('return banana') 262 | ir = C.ReturnIR.from_ast_node(stmt) 263 | frag = ir.as_fragment(translation_context) 264 | assert frag.n_states == 1 265 | succeed_state = frag.all_states[0] 266 | assert succeed_state.fields == {'Type': 'Succeed', 267 | 'InputPath': '$.locals.banana'} 268 | 269 | 270 | class TestRaiseIR: 271 | @pytest.fixture(scope='module', 272 | params=[C.RaiseIR.from_ast_node, mk_statement_empty_defs]) 273 | def factory(self, request): 274 | return request.param 275 | 276 | @pytest.fixture(scope='module') 277 | def sample_fail_stmt(self): 278 | return stmt_value('raise PSF.Fail("OverTemp", "too hot!")') 279 | 280 | def test_raise(self, factory, sample_fail_stmt): 281 | ir = factory(sample_fail_stmt) 282 | assert ir.error == 'OverTemp' 283 | assert ir.cause == 'too hot!' 284 | 285 | def test_raise_bad_input(self, factory): 286 | _test_factory_raises(stmt_value('raise x.y()'), factory) 287 | 288 | def test_as_fragment(self, sample_fail_stmt, translation_context): 289 | ir = C.RaiseIR.from_ast_node(sample_fail_stmt) 290 | frag = ir.as_fragment(translation_context) 291 | assert frag.n_states == 1 292 | assert len(frag.exit_states) == 0 293 | fail_state = frag.all_states[0] 294 | assert fail_state.fields == {'Type': 'Fail', 295 | 'Error': 'OverTemp', 296 | 'Cause': 'too hot!'} 297 | 298 | 299 | @pytest.fixture(scope='module') 300 | def sample_funcall_with_retry(): 301 | return expr_value('PSF.with_retry_spec(foo, (bar, baz),' 302 | ' (["Bad"], 1.5, 3, 1.5),' 303 | ' (["Worse"], 1.75, 5, 2.5))') 304 | 305 | 306 | class TestFunctionCallIR: 307 | @pytest.fixture(scope='module', 308 | params=[C.FunctionCallIR.from_ast_node, 309 | mk_assign_src_empty_defs]) 310 | def factory(self, request): 311 | return request.param 312 | 313 | def test_bare_call(self, factory): 314 | expr = expr_value('foo(bar, baz)') 315 | ir = factory(expr) 316 | assert ir.fun_name == 'foo' 317 | assert ir.arg_names == ['bar', 'baz'] 318 | assert ir.retry_spec is None 319 | 320 | def test_call_with_retry_spec(self, sample_funcall_with_retry, factory): 321 | ir = factory(sample_funcall_with_retry) 322 | assert ir.fun_name == 'foo' 323 | assert ir.arg_names == ['bar', 'baz'] 324 | assert ir.retry_spec[0].error_equals == ['Bad'] 325 | assert ir.retry_spec[0].interval_seconds == 1.5 326 | assert ir.retry_spec[0].max_attempts == 3 327 | assert ir.retry_spec[0].backoff_rate == 1.5 328 | assert ir.retry_spec[1].error_equals == ['Worse'] 329 | assert ir.retry_spec[1].interval_seconds == 1.75 330 | assert ir.retry_spec[1].max_attempts == 5 331 | assert ir.retry_spec[1].backoff_rate == 2.5 332 | 333 | def test_as_fragment(self, sample_funcall_with_retry, 334 | translation_context, factory): 335 | ir = factory(sample_funcall_with_retry) 336 | frag = ir.as_fragment(translation_context, 'the_result') 337 | assert frag.n_states == 2 338 | pass_state = frag.all_states[0] 339 | assert pass_state is frag.enter_state 340 | task_state = frag.all_states[1] 341 | assert task_state is frag.exit_states[0] 342 | 343 | # This doesn't check retry-spec but that's tested by 344 | # test_call_with_retry_spec(). 345 | _assert_state_pair_forms_assignment(pass_state, task_state, 346 | translation_context, 347 | 'the_result', 'foo', ['bar', 'baz']) 348 | 349 | 350 | class TestAssignmentIR: 351 | @pytest.fixture(scope='module', params=[C.AssignmentIR, C.StatementIR]) 352 | def assignment_class(self, request): 353 | return request.param 354 | 355 | def test_bare_call(self, assignment_class): 356 | stmt = stmt_value('foo = bar(baz, qux)') 357 | ir = assignment_class.from_ast_node(stmt, {}) 358 | _assert_is_assignment(ir, 'foo', 'bar', 'baz', 'qux') 359 | 360 | def test_as_fragment(self, translation_context): 361 | stmt = stmt_value('foo = bar(baz, qux)') 362 | ir = C.AssignmentIR.from_ast_node(stmt, {}) 363 | frag = ir.as_fragment(translation_context) 364 | assert frag.n_states == 2 365 | pass_state = frag.all_states[0] 366 | assert pass_state is frag.enter_state 367 | task_state = frag.all_states[1] 368 | assert task_state is frag.exit_states[0] 369 | assert pass_state.fields == {'Type': 'Pass', 370 | 'Result': {'function': 'bar', 371 | 'arg_names': ['baz', 'qux']}, 372 | 'ResultPath': '$.call_descr'} 373 | assert pass_state.next_state_name == task_state.name 374 | assert task_state.fields == {'Type': 'Task', 375 | 'Resource': translation_context.lambda_arn, 376 | 'ResultPath': '$.locals.foo'} 377 | 378 | 379 | class TestTryIR: 380 | @pytest.fixture(scope='module', 381 | params=[C.TryIR.from_ast_node, mk_statement_empty_defs]) 382 | def factory(self, request): 383 | return request.param 384 | 385 | def test_try(self, sample_try_stmt, factory): 386 | ir = factory(sample_try_stmt) 387 | _assert_is_assignment(ir.body.body[0], 'x', 'f', 'y') 388 | _assert_sample_try_catchers_correct(ir.catchers) 389 | 390 | def test_as_fragment(self, sample_try_stmt, translation_context): 391 | ir = C.TryIR.from_ast_node(sample_try_stmt) 392 | frag = ir.as_fragment(translation_context) 393 | assert frag.n_states == 10 # Two per assignment 394 | s0 = frag.enter_state 395 | s1 = find_successor_state(frag, s0) 396 | _assert_state_pair_forms_assignment(s0, s1, 397 | translation_context, 398 | 'x', 'f', ['y']) 399 | catches = s1.fields['Catch'] 400 | assert len(catches) == 2 401 | assert catches[0]['ErrorEquals'] == ['BadThing'] 402 | catch0_s0 = find_state_by_name(frag, catches[0]['Next']) 403 | catch0_s1 = find_successor_state(frag, catch0_s0) 404 | _assert_state_pair_forms_assignment(catch0_s0, catch0_s1, 405 | translation_context, 406 | 'foo', 'bar', ['baz']) 407 | assert catches[1]['ErrorEquals'] == ['WorseThing'] 408 | catch1_s0 = find_state_by_name(frag, catches[1]['Next']) 409 | catch1_s1 = find_successor_state(frag, catch1_s0) 410 | _assert_state_pair_forms_assignment(catch1_s0, catch1_s1, 411 | translation_context, 412 | 'qux', 'bar', ['baz']) 413 | 414 | 415 | @pytest.fixture(scope='module') 416 | def sample_if_statement(): 417 | return stmt_value(""" 418 | if PSF.StringEquals(foo, 'hello'): 419 | x = f(y) 420 | else: 421 | z = g(u) 422 | s = h(t) 423 | """) 424 | 425 | 426 | class TestIfIR: 427 | @pytest.fixture(scope='module', 428 | params=[C.IfIR.from_ast_node, mk_statement_empty_defs]) 429 | def factory(self, request): 430 | return request.param 431 | 432 | def test_if(self, sample_if_statement, factory): 433 | ir = factory(sample_if_statement) 434 | _assert_comparison_correct(ir.test, 'StringEquals', ['foo'], 'hello') 435 | _assert_is_assignment(ir.true_body.body[0], 'x', 'f', 'y') 436 | _assert_is_assignment(ir.false_body.body[0], 'z', 'g', 'u') 437 | _assert_is_assignment(ir.false_body.body[1], 's', 'h', 't') 438 | 439 | def test_as_fragment(self, translation_context, sample_if_statement): 440 | ir = C.IfIR.from_ast_node(sample_if_statement) 441 | frag = ir.as_fragment(translation_context) 442 | assert frag.n_states == 7 # Two per assignment; one for choice. 443 | assert len(frag.exit_states) == 2 # One per branch. 444 | assert frag.enter_state.fields['Type'] == 'Choice' 445 | choices = frag.enter_state.fields['Choices'] 446 | assert len(choices) == 1 447 | true_branch_s0 = find_state_by_name(frag, choices[0]['Next']) 448 | true_branch_s1 = find_successor_state(frag, true_branch_s0) 449 | _assert_state_pair_forms_assignment(true_branch_s0, true_branch_s1, 450 | translation_context, 451 | 'x', 'f', ['y']) 452 | false_branch_s0 = find_state_by_name(frag, frag.enter_state.fields['Default']) 453 | false_branch_s1 = find_successor_state(frag, false_branch_s0) 454 | _assert_state_pair_forms_assignment(false_branch_s0, false_branch_s1, 455 | translation_context, 456 | 'z', 'g', ['u']) 457 | 458 | 459 | @pytest.fixture(scope='module') 460 | def sample_parallel_invocation(): 461 | return suite_value(""" 462 | def f1(): 463 | r = f(bar, baz) 464 | s = g(r) 465 | return s 466 | def f2(): 467 | x = m(u) 468 | return x 469 | results = PSF.parallel(f1, f2) 470 | """) 471 | 472 | 473 | class TestParallelIR: 474 | @staticmethod 475 | def _assert_parallel_ir_correct(ir): 476 | assert len(ir.branches) == 2 477 | br0 = ir.branches[0] 478 | _assert_is_assignment(br0.body[0], 'r', 'f', 'bar', 'baz') 479 | _assert_is_assignment(br0.body[1], 's', 'g', 'r') 480 | _assert_is_return(br0.body[2], 's') 481 | br1 = ir.branches[1] 482 | _assert_is_assignment(br1.body[0], 'x', 'm', 'u') 483 | _assert_is_return(br1.body[1], 'x') 484 | 485 | def test_parallel(self, sample_parallel_invocation): 486 | def_f1 = C.SuiteIR.from_ast_nodes(sample_parallel_invocation[0].body) 487 | def_f2 = C.SuiteIR.from_ast_nodes(sample_parallel_invocation[1].body) 488 | ir = C.ParallelIR.from_ast_node_and_defs( 489 | sample_parallel_invocation[2].value, 490 | {'f1': def_f1, 'f2': def_f2}) 491 | self._assert_parallel_ir_correct(ir) 492 | 493 | def test_parallel_assignment(self, sample_parallel_invocation): 494 | ir = C.SuiteIR.from_ast_nodes(sample_parallel_invocation) 495 | assert len(ir.body) == 1 496 | assert isinstance(ir.body[0], C.AssignmentIR) 497 | assert isinstance(ir.body[0].source, C.ParallelIR) 498 | self._assert_parallel_ir_correct(ir.body[0].source) 499 | 500 | def test_as_fragment(self, 501 | translation_context, sample_parallel_invocation): 502 | ir = C.SuiteIR.from_ast_nodes(sample_parallel_invocation) 503 | parallel_ir = ir.body[0].source 504 | frag = parallel_ir.as_fragment(translation_context, 'the_results') 505 | assert frag.n_states == 1 506 | branches = frag.enter_state.fields['Branches'] 507 | assert len(branches) == 2 508 | b0, b1 = branches 509 | assert len(b0['States']) == 5 # Two per assignment ... 510 | assert len(b1['States']) == 3 # ... plus one return 511 | 512 | 513 | class TestSuiteIR: 514 | @pytest.fixture(scope='module') 515 | def sample_suite(self): 516 | return suite_value(""" 517 | foo = bar(baz) 518 | qux = hello(world) 519 | """) 520 | 521 | def test_assignments(self, sample_suite): 522 | ir = C.SuiteIR.from_ast_nodes(sample_suite) 523 | _assert_is_assignment(ir.body[0], 'foo', 'bar', 'baz') 524 | _assert_is_assignment(ir.body[1], 'qux', 'hello', 'world') 525 | 526 | def test_as_fragment(self, sample_suite, translation_context): 527 | ir = C.SuiteIR.from_ast_nodes(sample_suite) 528 | frag = ir.as_fragment(translation_context) 529 | assert frag.n_states == 4 # Two per assignment 530 | states = frag.all_states 531 | assert frag.enter_state is states[0] 532 | _assert_state_pair_forms_assignment(states[0], states[1], 533 | translation_context, 534 | 'foo', 'bar', ['baz']) 535 | assert states[1].next_state_name == states[2].name 536 | _assert_state_pair_forms_assignment(states[2], states[3], 537 | translation_context, 538 | 'qux', 'hello', ['world']) 539 | assert frag.exit_states == [states[3]] 540 | 541 | 542 | class TestStateMachineStateIR: 543 | def test_construction(self): 544 | sms_1 = C.StateMachineStateIR.from_fields(Type='Wait', Seconds=30) 545 | sms_2 = C.StateMachineStateIR.from_fields(Type='Wait', Seconds=60) 546 | assert sms_1.name != sms_2.name 547 | assert sms_1.fields == {'Type': 'Wait', 'Seconds': 30} 548 | assert sms_2.fields == {'Type': 'Wait', 'Seconds': 60} 549 | 550 | def test_as_json_no_next(self): 551 | sms = C.StateMachineStateIR.from_fields(Type='Wait', Seconds=30) 552 | assert sms.value_as_json_obj() == {'Type': 'Wait', 'Seconds': 30} 553 | 554 | def test_as_json_with_next(self): 555 | sms = C.StateMachineStateIR.from_fields(Type='Wait', Seconds=30) 556 | sms.next_state_name = 'do_something' 557 | assert sms.value_as_json_obj() == {'Type': 'Wait', 'Seconds': 30, 558 | 'Next': 'do_something'} 559 | --------------------------------------------------------------------------------