├── .coveragerc ├── .github └── workflows │ └── test.yml ├── .gitignore ├── LICENSE ├── MANIFEST.in ├── README.md ├── ckanserviceprovider ├── __init__.py ├── db.py ├── default_settings.py ├── job.py ├── util.py └── web.py ├── doc ├── Makefile ├── _themes │ ├── LICENSE │ ├── flask_small │ │ ├── layout.html │ │ ├── static │ │ │ └── flasky.css_t │ │ └── theme.conf │ └── flask_theme_support.py ├── conf.py └── index.rst ├── example ├── __init__.py ├── jobs.py └── main.py ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── settings_test.py ├── test_db.py ├── test_joberror.py └── test_web.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = ckanserviceprovider 3 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | on: [push, pull_request] 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] 8 | fail-fast: false 9 | name: Python ${{ matrix.python-version }} 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | - uses: actions/setup-python@v4 14 | with: 15 | python-version: ${{ matrix.python-version }} 16 | - name: Install requirements (Python 3) 17 | run: pip install -r requirements-dev.txt && pip install . 18 | - name: Run tests 19 | run: pytest --cov=ckanserviceprovider --cov-append --cov-report=xml --disable-warnings tests 20 | - name: Upload coverage report to codecov 21 | uses: codecov/codecov-action@v3 22 | with: 23 | file: ./coverage.xml 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | syntax: glob 2 | # generic 3 | *.pyc 4 | *.swp 5 | *.swo 6 | ckanserviceprovider.egg-info/* 7 | *.db 8 | settings_local.py 9 | dist/* 10 | build/* 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published 637 | by the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Tests](https://github.com/ckan/ckan-service-provider/actions/workflows/test.yml/badge.svg)](https://github.com/ckan/ckan-service-provider/actions/workflows/test.yml) 2 | [![Latest Version](https://img.shields.io/pypi/v/ckanserviceprovider.svg)](https://pypi.python.org/pypi/ckanserviceprovider/) 3 | [![Downloads](https://img.shields.io/pypi/dm/ckanserviceprovider.svg)](https://pypi.python.org/pypi/ckanserviceprovider/) 4 | [![Supported Python versions](https://img.shields.io/pypi/pyversions/ckanserviceprovider.svg)](https://pypi.python.org/pypi/ckanserviceprovider/) 5 | [![Development Status](https://img.shields.io/pypi/status/ckanserviceprovider.svg)](https://pypi.python.org/pypi/ckanserviceprovider/) 6 | [![License](https://img.shields.io/pypi/l/ckanserviceprovider.svg)](https://pypi.python.org/pypi/ckanserviceprovider/) 7 | 8 | [DataPusher]: https://github.com/okfn/datapusher 9 | [PyPI]: https://pypi.python.org/pypi/ckanserviceprovider 10 | 11 | 12 | # CKAN Service Provider 13 | 14 | A library for making web services that make functions available as synchronous 15 | or asynchronous jobs. Used by [DataPusher][]. 16 | 17 | 18 | ## Getting Started 19 | 20 | To install ckanserviceprovider for development: 21 | 22 | ```bash 23 | git clone https://github.com/ckan/ckan-service-provider.git 24 | cd ckan-service-provider 25 | pip install -r requirements-dev.txt 26 | ``` 27 | 28 | To get started making a web service with ckanserviceprovider have a look at 29 | [/example](example). You can run the example server with 30 | `python example/main.py example/settings_local.py`. 31 | 32 | For a real-world example have a look at [DataPusher][]. 33 | 34 | 35 | ## Running the Tests 36 | 37 | To run the ckanserviceprovider tests: 38 | 39 | ```bash 40 | pytest 41 | ``` 42 | 43 | 44 | ## Building the Documentation 45 | 46 | To build the ckanserviceprovider docs: 47 | 48 | ```bash 49 | python setup.py build_sphinx 50 | ``` 51 | 52 | 53 | ## Releasing a New Version 54 | 55 | To release a new version of ckanserviceprovider: 56 | 57 | 1. Increment the version number in [setup.py](setup.py) 58 | 59 | 2. Build a source distribution of the new version and publish it to 60 | [PyPI][]: 61 | 62 | ```bash 63 | python setup.py sdist bdist_wheel 64 | pip install --upgrade twine 65 | twine upload dist/* 66 | ``` 67 | 68 | You may want to test installing and running the new version from PyPI in a 69 | clean virtualenv before continuing to the next step. 70 | 71 | 3. Commit your setup.py changes to git, tag the release, and push the changes 72 | and the tag to GitHub: 73 | 74 | ```bash 75 | git commit setup.py -m "Bump version number" 76 | git tag 0.0.1 77 | git push 78 | git push origin 0.0.1 79 | ``` 80 | 81 | (Replace both instances of 0.0.1 with the number of the version you're 82 | releasing.) 83 | 84 | 85 | ## Authors 86 | 87 | The original authors of ckanserviceprovider were 88 | David Raznick and 89 | Dominik Moritz . For the current list of contributors 90 | see [github.com/ckan/ckan-service-provider/contributors](https://github.com/ckan/ckan-service-provider/contributors) 91 | -------------------------------------------------------------------------------- /ckanserviceprovider/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ckan/ckan-service-provider/3f7fc99a9fbed5bbac064b3068887aff97adfcc3/ckanserviceprovider/__init__.py -------------------------------------------------------------------------------- /ckanserviceprovider/db.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """A module that encapsulates CKAN Service Provider's model/database. 3 | 4 | This module provides a set of public functions that other modules should call 5 | to interact with the database, rather than using sqlalchemy directly. 6 | 7 | TODO: Some more refactoring is still needed to have the model completely 8 | encapsulated in this module, other modules are still using sqlalchemy directly 9 | in some places. 10 | 11 | The database contains: 12 | 13 | 1. A jobs table that tracks the statuses of pending and completed jobs 14 | 2. A metdata table that contains (key, value) metadata pairs associated with 15 | jobs 16 | 3. A logs table that contains log output from jobs. 17 | 18 | This module's public interface includes: 19 | 20 | init() 21 | Initialise the database connection on application or test startup, creating the 22 | database and tables if necessary. 23 | 24 | drop_all() 25 | Delete all the database tables, for tests. 26 | 27 | get_job() 28 | Get a dictionary representation of a job from the database. 29 | 30 | add_pending_job() 31 | Add a new job with status "pending" to the database. 32 | Jobs always have status "pending" when they're first added. 33 | 34 | mark_job_as_*() 35 | A set of functions for updating an existing job in the database. 36 | There are a limited number of ways in which CKAN Service Provider updates jobs. 37 | For example marking a job as completed successfully, or marking a job as 38 | failed with an error. These mark_job_as_*() functions define all the ways that 39 | a job can be updated. 40 | 41 | See the functions' own docstrings for details. 42 | 43 | """ 44 | 45 | import datetime 46 | import json 47 | 48 | import sqlalchemy 49 | 50 | 51 | # Some module-global constants. Some of these are accessed directly by other 52 | # modules. It would be good to factor these out. 53 | ENGINE = None 54 | _METADATA = None 55 | JOBS_TABLE = None 56 | METADATA_TABLE = None 57 | LOGS_TABLE = None 58 | 59 | 60 | def init(uri, echo=False): 61 | """Initialise the database. 62 | 63 | Initialise the sqlalchemy engine, metadata and table objects that we use to 64 | connect to the database. 65 | 66 | Create the database and the database tables themselves if they don't 67 | already exist. 68 | 69 | :param uri: the sqlalchemy database URI 70 | :type uri: string 71 | 72 | :param echo: whether or not to have the sqlalchemy engine log all 73 | statements to stdout 74 | :type echo: bool 75 | 76 | """ 77 | global ENGINE, _METADATA, JOBS_TABLE, METADATA_TABLE, LOGS_TABLE 78 | ENGINE = sqlalchemy.create_engine(uri, echo=echo, convert_unicode=True) 79 | _METADATA = sqlalchemy.MetaData(ENGINE) 80 | JOBS_TABLE = _init_jobs_table() 81 | METADATA_TABLE = _init_metadata_table() 82 | LOGS_TABLE = _init_logs_table() 83 | _METADATA.create_all(ENGINE) 84 | 85 | 86 | def drop_all(): 87 | """Delete all the database tables (if they exist). 88 | 89 | This is for tests to reset the DB. Note that this will delete *all* tables 90 | in the database, not just tables created by this module (for example 91 | apscheduler's tables will also be deleted). 92 | 93 | """ 94 | if _METADATA: 95 | _METADATA.drop_all(ENGINE) 96 | 97 | 98 | def get_job(job_id, limit=None, use_aps_id=False): 99 | """Return the job with the given job_id as a dict. 100 | 101 | The dict also includes any metadata or logs associated with the job. 102 | 103 | Returns None instead of a dict if there's no job with the given job_id. 104 | 105 | The keys of a job dict are: 106 | 107 | "job_id": The unique identifier for the job (unicode) 108 | 109 | "job_type": The name of the job function that will be executed for this 110 | job (unicode) 111 | 112 | "status": The current status of the job, e.g. "pending", "complete", or 113 | "error" (unicode) 114 | 115 | "data": Any output data returned by the job if it has completed 116 | successfully. This may be any JSON-serializable type, e.g. None, a 117 | string, a dict, etc. 118 | 119 | "error": If the job failed with an error this will be a dict with a 120 | "message" key whose value is a string error message. The dict may also 121 | have other keys specific to the particular type of error. If the job 122 | did not fail with an error then "error" will be None. 123 | 124 | "requested_timestamp": The time at which the job was requested (string) 125 | 126 | "finished_timestamp": The time at which the job finished (string) 127 | 128 | "sent_data": The input data for the job, provided by the client site. 129 | This may be any JSON-serializable type, e.g. None, a string, a dict, 130 | etc. 131 | 132 | "result_url": The callback URL that CKAN Service Provider will post the 133 | result to when the job finishes (unicode) 134 | 135 | "api_key": The API key that CKAN Service Provider will use when posting 136 | the job result to the result_url (unicode or None). A None here doesn't 137 | mean that there was no API key: CKAN Service Provider deletes the API 138 | key from the database after it has posted the result to the result_url. 139 | 140 | "job_key": The key that users must provide (in the Authorization header of 141 | the HTTP request) to be authorized to modify the job (unicode). 142 | For example requests to the CKAN Service Provider API need this to get 143 | the status or output data of a job or to delete a job. 144 | If you login to CKAN Service Provider as an administrator then you can 145 | administer any job without providing its job_key. 146 | 147 | "metadata": Any custom metadata associated with the job (dict) 148 | 149 | "logs": Any logs associated with the job (list) 150 | 151 | """ 152 | # Avoid SQLAlchemy "Unicode type received non-unicode bind param value" 153 | # warnings. 154 | if job_id: 155 | job_id = str(job_id) 156 | 157 | if use_aps_id: 158 | result = ENGINE.execute( 159 | JOBS_TABLE.select().where(JOBS_TABLE.c.aps_job_id == job_id) 160 | ).first() 161 | else: 162 | result = ENGINE.execute( 163 | JOBS_TABLE.select().where(JOBS_TABLE.c.job_id == job_id) 164 | ).first() 165 | 166 | if not result: 167 | return None 168 | 169 | # Turn the result into a dictionary representation of the job. 170 | result_dict = {} 171 | for field in list(result.keys()): 172 | value = getattr(result, field) 173 | if value is None: 174 | result_dict[field] = value 175 | elif field in ("sent_data", "data", "error"): 176 | result_dict[field] = json.loads(value) 177 | elif isinstance(value, datetime.datetime): 178 | result_dict[field] = value.isoformat() 179 | else: 180 | result_dict[field] = str(value) 181 | 182 | result_dict["metadata"] = _get_metadata(job_id) 183 | result_dict["logs"] = _get_logs(job_id, limit=limit) 184 | 185 | return result_dict 186 | 187 | 188 | def add_pending_job( 189 | job_id, job_key, job_type, api_key, data=None, metadata=None, result_url=None 190 | ): 191 | """Add a new job with status "pending" to the jobs table. 192 | 193 | All code that adds jobs to the jobs table should go through this function. 194 | Code that adds to the jobs table manually should be refactored to use this 195 | function. 196 | 197 | May raise unspecified exceptions from Python core, SQLAlchemy or JSON! 198 | TODO: Document and unit test these! 199 | 200 | :param job_id: a unique identifier for the job, used as the primary key in 201 | ckanserviceprovider's "jobs" database table 202 | :type job_id: unicode 203 | 204 | :param job_key: the key required to administer the job via the API 205 | :type job_key: unicode 206 | 207 | :param job_type: the name of the job function that will be executed for 208 | this job 209 | :type job_key: unicode 210 | 211 | :param api_key: the client site API key that ckanserviceprovider will use 212 | when posting the job result to the result_url 213 | :type api_key: unicode 214 | 215 | :param data: The input data for the job (called sent_data elsewhere) 216 | :type data: Any JSON-serializable type 217 | 218 | :param metadata: A dict of arbitrary (key, value) metadata pairs to be 219 | stored along with the job. The keys should be strings, the values can 220 | be strings or any JSON-encodable type. 221 | :type metadata: dict 222 | 223 | :param result_url: the callback URL that ckanserviceprovider will post the 224 | job result to when the job has finished 225 | :type result_url: unicode 226 | 227 | """ 228 | if not data: 229 | data = {} 230 | data = json.dumps(data) 231 | 232 | # Turn strings into unicode to stop SQLAlchemy 233 | # "Unicode type received non-unicode bind param value" warnings. 234 | if job_id: 235 | job_id = str(job_id) 236 | if job_type: 237 | job_type = str(job_type) 238 | if result_url: 239 | result_url = str(result_url) 240 | if api_key: 241 | api_key = str(api_key) 242 | if job_key: 243 | job_key = str(job_key) 244 | data = str(data) 245 | 246 | if not metadata: 247 | metadata = {} 248 | 249 | conn = ENGINE.connect() 250 | trans = conn.begin() 251 | try: 252 | conn.execute( 253 | JOBS_TABLE.insert().values( 254 | job_id=job_id, 255 | job_type=job_type, 256 | status="pending", 257 | requested_timestamp=datetime.datetime.now(), 258 | sent_data=data, 259 | result_url=result_url, 260 | api_key=api_key, 261 | job_key=job_key, 262 | ) 263 | ) 264 | 265 | # Insert any (key, value) metadata pairs that the job has into the 266 | # metadata table. 267 | inserts = [] 268 | for key, value in list(metadata.items()): 269 | type_ = "string" 270 | if not isinstance(value, str): 271 | value = json.dumps(value) 272 | type_ = "json" 273 | 274 | # Turn strings into unicode to stop SQLAlchemy 275 | # "Unicode type received non-unicode bind param value" warnings. 276 | key = str(key) 277 | value = str(value) 278 | 279 | inserts.append( 280 | {"job_id": job_id, "key": key, "value": value, "type": type_} 281 | ) 282 | if inserts: 283 | conn.execute(METADATA_TABLE.insert(), inserts) 284 | trans.commit() 285 | except Exception: 286 | trans.rollback() 287 | raise 288 | finally: 289 | conn.close() 290 | 291 | 292 | class InvalidErrorObjectError(Exception): 293 | pass 294 | 295 | 296 | def _validate_error(error): 297 | """Validate and return the given error object. 298 | 299 | Based on the given error object, return either None or a dict with a 300 | "message" key whose value is a string (the dict may also have any other 301 | keys that it wants). 302 | 303 | The given "error" object can be: 304 | 305 | - None, in which case None is returned 306 | 307 | - A string, in which case a dict like this will be returned: 308 | {"message": error_string} 309 | 310 | - A dict with a "message" key whose value is a string, in which case the 311 | dict will be returned unchanged 312 | 313 | :param error: the error object to validate 314 | 315 | :raises InvalidErrorObjectError: If the error object doesn't match any of 316 | the allowed types 317 | 318 | """ 319 | if error is None: 320 | return None 321 | elif isinstance(error, str): 322 | return {"message": error} 323 | else: 324 | try: 325 | message = error["message"] 326 | if isinstance(message, str): 327 | return error 328 | else: 329 | raise InvalidErrorObjectError("error['message'] must be a string") 330 | except (TypeError, KeyError): 331 | raise InvalidErrorObjectError( 332 | "error must be either a string or a dict with a message key" 333 | ) 334 | 335 | 336 | def _update_job(job_id, job_dict): 337 | """Update the database row for the given job_id with the given job_dict. 338 | 339 | All functions that update rows in the jobs table do it by calling this 340 | helper function. 341 | 342 | job_dict is a dict with values corresponding to the database columns that 343 | should be updated, e.g.: 344 | 345 | {"status": "complete", "data": ...} 346 | 347 | """ 348 | # Avoid SQLAlchemy "Unicode type received non-unicode bind param value" 349 | # warnings. 350 | if job_id: 351 | job_id = str(job_id) 352 | 353 | if "error" in job_dict: 354 | job_dict["error"] = _validate_error(job_dict["error"]) 355 | job_dict["error"] = json.dumps(job_dict["error"]) 356 | # Avoid SQLAlchemy "Unicode type received non-unicode bind param value" 357 | # warnings. 358 | job_dict["error"] = str(job_dict["error"]) 359 | 360 | # Avoid SQLAlchemy "Unicode type received non-unicode bind param value" 361 | # warnings. 362 | if "data" in job_dict: 363 | job_dict["data"] = str(job_dict["data"]) 364 | 365 | ENGINE.execute( 366 | JOBS_TABLE.update().where(JOBS_TABLE.c.job_id == job_id).values(**job_dict) 367 | ) 368 | 369 | 370 | def mark_job_as_completed(job_id, data=None): 371 | """Mark a job as completed successfully. 372 | 373 | :param job_id: the job_id of the job to be updated 374 | :type job_id: unicode 375 | 376 | :param data: the output data returned by the job 377 | :type data: any JSON-serializable type (including None) 378 | 379 | """ 380 | update_dict = { 381 | "status": "complete", 382 | "data": json.dumps(data), 383 | "finished_timestamp": datetime.datetime.now(), 384 | } 385 | _update_job(job_id, update_dict) 386 | 387 | 388 | def mark_job_as_missed(job_id): 389 | """Mark a job as missed because it was in the queue for too long. 390 | 391 | :param job_id: the job_id of the job to be updated 392 | :type job_id: unicode 393 | 394 | """ 395 | update_dict = { 396 | "status": "error", 397 | "error": "Job delayed too long, service full", 398 | "finished_timestamp": datetime.datetime.now(), 399 | } 400 | _update_job(job_id, update_dict) 401 | 402 | 403 | def mark_job_as_errored(job_id, error_object): 404 | """Mark a job as failed with an error. 405 | 406 | :param job_id: the job_id of the job to be updated 407 | :type job_id: unicode 408 | 409 | :param error_object: the error returned by the job 410 | :type error_object: either a string or a dict with a "message" key whose 411 | value is a string 412 | 413 | """ 414 | update_dict = { 415 | "status": "error", 416 | "error": error_object, 417 | "finished_timestamp": datetime.datetime.now(), 418 | } 419 | _update_job(job_id, update_dict) 420 | 421 | 422 | def mark_job_as_failed_to_post_result(job_id): 423 | """Mark a job as 'failed to post result'. 424 | 425 | This happens when a job completes (either successfully or with an error) 426 | then trying to post the job result back to the job's callback URL fails. 427 | 428 | FIXME: This overwrites any error from the job itself! 429 | 430 | :param job_id: the job_id of the job to be updated 431 | :type job_id: unicode 432 | 433 | """ 434 | update_dict = { 435 | "error": "Process completed but unable to post to result_url", 436 | } 437 | _update_job(job_id, update_dict) 438 | 439 | 440 | def delete_api_key(job_id): 441 | """Delete the given job's API key from the database. 442 | 443 | The API key is used when posting the job's result to the client's callback 444 | URL. This function should be called to delete the API key after the result 445 | has been posted - the API key is no longer needed. 446 | 447 | """ 448 | _update_job(job_id, {"api_key": None}) 449 | 450 | 451 | def set_aps_job_id(job_id, aps_job_id): 452 | 453 | _update_job(job_id, {"aps_job_id": aps_job_id}) 454 | 455 | 456 | def _init_jobs_table(): 457 | """Initialise the "jobs" table in the db.""" 458 | _jobs_table = sqlalchemy.Table( 459 | "jobs", 460 | _METADATA, 461 | sqlalchemy.Column("job_id", sqlalchemy.UnicodeText, primary_key=True), 462 | sqlalchemy.Column("job_type", sqlalchemy.UnicodeText), 463 | sqlalchemy.Column("status", sqlalchemy.UnicodeText, index=True), 464 | sqlalchemy.Column("data", sqlalchemy.UnicodeText), 465 | sqlalchemy.Column("error", sqlalchemy.UnicodeText), 466 | sqlalchemy.Column("requested_timestamp", sqlalchemy.DateTime), 467 | sqlalchemy.Column("finished_timestamp", sqlalchemy.DateTime), 468 | sqlalchemy.Column("sent_data", sqlalchemy.UnicodeText), 469 | sqlalchemy.Column("aps_job_id", sqlalchemy.UnicodeText), 470 | # Callback URL: 471 | sqlalchemy.Column("result_url", sqlalchemy.UnicodeText), 472 | # CKAN API key: 473 | sqlalchemy.Column("api_key", sqlalchemy.UnicodeText), 474 | # Key to administer job: 475 | sqlalchemy.Column("job_key", sqlalchemy.UnicodeText), 476 | ) 477 | return _jobs_table 478 | 479 | 480 | def _init_metadata_table(): 481 | """Initialise the "metadata" table in the db.""" 482 | _metadata_table = sqlalchemy.Table( 483 | "metadata", 484 | _METADATA, 485 | sqlalchemy.Column( 486 | "job_id", 487 | sqlalchemy.ForeignKey("jobs.job_id", ondelete="CASCADE"), 488 | nullable=False, 489 | primary_key=True, 490 | ), 491 | sqlalchemy.Column("key", sqlalchemy.UnicodeText, primary_key=True), 492 | sqlalchemy.Column("value", sqlalchemy.UnicodeText, index=True), 493 | sqlalchemy.Column("type", sqlalchemy.UnicodeText), 494 | ) 495 | return _metadata_table 496 | 497 | 498 | def _init_logs_table(): 499 | """Initialise the "logs" table in the db.""" 500 | _logs_table = sqlalchemy.Table( 501 | "logs", 502 | _METADATA, 503 | sqlalchemy.Column( 504 | "job_id", 505 | sqlalchemy.ForeignKey("jobs.job_id", ondelete="CASCADE"), 506 | nullable=False, 507 | ), 508 | sqlalchemy.Column("timestamp", sqlalchemy.DateTime), 509 | sqlalchemy.Column("message", sqlalchemy.UnicodeText), 510 | sqlalchemy.Column("level", sqlalchemy.UnicodeText), 511 | sqlalchemy.Column("module", sqlalchemy.UnicodeText), 512 | sqlalchemy.Column("funcName", sqlalchemy.UnicodeText), 513 | sqlalchemy.Column("lineno", sqlalchemy.Integer), 514 | ) 515 | return _logs_table 516 | 517 | 518 | def _get_metadata(job_id): 519 | """Return any metadata for the given job_id from the metadata table.""" 520 | # Avoid SQLAlchemy "Unicode type received non-unicode bind param value" 521 | # warnings. 522 | job_id = str(job_id) 523 | 524 | results = ENGINE.execute( 525 | METADATA_TABLE.select().where(METADATA_TABLE.c.job_id == job_id) 526 | ).fetchall() 527 | metadata = {} 528 | for row in results: 529 | value = row["value"] 530 | if row["type"] == "json": 531 | value = json.loads(value) 532 | metadata[row["key"]] = value 533 | return metadata 534 | 535 | 536 | def _get_logs(job_id, limit=None): 537 | """Return any logs for the given job_id from the logs table.""" 538 | # Avoid SQLAlchemy "Unicode type received non-unicode bind param value" 539 | # warnings. 540 | job_id = str(job_id) 541 | try: 542 | int(limit) 543 | limit_is_valid = True 544 | except (ValueError, TypeError): 545 | # None or "one" (or similar) 546 | limit_is_valid = False 547 | 548 | if not limit_is_valid: 549 | results = ENGINE.execute( 550 | LOGS_TABLE.select().where(LOGS_TABLE.c.job_id == job_id) 551 | ).fetchall() 552 | else: 553 | results = ENGINE.execute( 554 | LOGS_TABLE.select() 555 | .where(LOGS_TABLE.c.job_id == job_id) 556 | .order_by(LOGS_TABLE.c.timestamp.desc()) 557 | .limit(limit) 558 | ).fetchall() 559 | 560 | results = [dict(result) for result in results] 561 | 562 | for result in results: 563 | result.pop("job_id") 564 | 565 | return results 566 | 567 | 568 | def add_logs(job_id, message=None, level=None, module=None, funcName=None, lineno=None): 569 | """for tests only""" 570 | if job_id: 571 | job_id = str(job_id) 572 | if message: 573 | message = str(message) 574 | if level: 575 | level = str(level) 576 | if module: 577 | module = str(module) 578 | if funcName: 579 | funcName = str(funcName) 580 | if lineno: 581 | lineno = str(lineno) 582 | conn = ENGINE.connect() 583 | trans = conn.begin() 584 | 585 | conn.execute( 586 | LOGS_TABLE.insert().values( 587 | job_id=job_id, 588 | timestamp=datetime.datetime.now(), 589 | message=message, 590 | level=level, 591 | module=module, 592 | funcName=funcName, 593 | lineno=lineno, 594 | ) 595 | ) 596 | trans.commit() 597 | conn.close() 598 | -------------------------------------------------------------------------------- /ckanserviceprovider/default_settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | DEBUG = False 4 | TESTING = False 5 | SECRET_KEY = "please_replace_me" 6 | USERNAME = "admin" 7 | PASSWORD = "pass" 8 | 9 | # database 10 | 11 | SQLALCHEMY_DATABASE_URI = os.environ.get("DATABASE_URL", "sqlite://") 12 | SQLALCHEMY_ECHO = False 13 | 14 | # webserver host and port 15 | 16 | HOST = "0.0.0.0" 17 | PORT = 8000 18 | 19 | # logging 20 | 21 | LOG_FILE = "/tmp/ckan_service.log" 22 | STDERR = True 23 | 24 | # project configuration 25 | NAME = "service" 26 | 27 | # days to keep when clearing 28 | KEEP_JOBS_AGE = 14 29 | -------------------------------------------------------------------------------- /ckanserviceprovider/job.py: -------------------------------------------------------------------------------- 1 | from . import web 2 | 3 | 4 | def asynchronous(func): 5 | web.async_types[func.__name__] = func 6 | return func 7 | 8 | 9 | def synchronous(func): 10 | web.sync_types[func.__name__] = func 11 | return func 12 | -------------------------------------------------------------------------------- /ckanserviceprovider/util.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | import logging 5 | import datetime 6 | from future.utils import python_2_unicode_compatible 7 | 8 | from . import db 9 | 10 | 11 | @python_2_unicode_compatible 12 | class JobError(Exception): 13 | """The exception type that jobs raise to signal failure.""" 14 | 15 | def __init__(self, message): 16 | """Initialize a JobError with the given error message string. 17 | 18 | The error message string that you give here will be returned to the 19 | client site in the job dict's "error" key. 20 | 21 | """ 22 | self.message = message 23 | 24 | def as_dict(self): 25 | """Return a dictionary representation of this JobError object. 26 | 27 | Returns a dictionary with a "message" key whose value is a string error 28 | message - suitable for use as the "error" key in a ckanserviceprovider 29 | job dict. 30 | 31 | """ 32 | return {"message": self.message} 33 | 34 | def __str__(self): 35 | return self.message 36 | 37 | 38 | class StoringHandler(logging.Handler): 39 | """A handler that stores the logging records 40 | in the database.""" 41 | 42 | def __init__(self, task_id, input): 43 | logging.Handler.__init__(self) 44 | self.task_id = task_id 45 | self.input = input 46 | 47 | def emit(self, record): 48 | conn = db.ENGINE.connect() 49 | try: 50 | # Turn strings into unicode to stop SQLAlchemy 51 | # "Unicode type received non-unicode bind param value" warnings. 52 | message = str(record.getMessage()) 53 | level = str(record.levelname) 54 | module = str(record.module) 55 | funcName = str(record.funcName) 56 | 57 | conn.execute( 58 | db.LOGS_TABLE.insert().values( 59 | job_id=self.task_id, 60 | timestamp=datetime.datetime.now(), 61 | message=message, 62 | level=level, 63 | module=module, 64 | funcName=funcName, 65 | lineno=record.lineno, 66 | ) 67 | ) 68 | finally: 69 | conn.close() 70 | -------------------------------------------------------------------------------- /ckanserviceprovider/web.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | import uuid 5 | import datetime 6 | import sys 7 | import json 8 | import traceback 9 | import logging 10 | import logging.handlers 11 | 12 | import flask 13 | import flask_login as flogin 14 | import werkzeug 15 | from apscheduler.schedulers.background import BackgroundScheduler as Scheduler 16 | import apscheduler.events as events 17 | import apscheduler.jobstores.sqlalchemy as sqlalchemy_store 18 | from apscheduler.triggers.date import DateTrigger 19 | import sqlalchemy.sql as sql 20 | import sqlalchemy as sa 21 | import requests 22 | from werkzeug.middleware.proxy_fix import ProxyFix 23 | 24 | from . import db 25 | from . import util 26 | from . import default_settings 27 | 28 | # Some module-global constants. Some of these are accessed directly by other 29 | # modules. It would be good to factor out as many of these as possible. 30 | sync_types = {} 31 | async_types = {} 32 | job_statuses = ["pending", "complete", "error"] 33 | app = flask.Flask(__name__) 34 | scheduler = None 35 | _users = None 36 | _names = None 37 | SSL_VERIFY = None 38 | 39 | 40 | def init(): 41 | """Initialise and configure the app, database, scheduler, etc. 42 | 43 | This should be called once at application startup or at tests startup 44 | (and not e.g. called once for each test case). 45 | 46 | """ 47 | global _users, _names 48 | _configure_app(app) 49 | _users, _names = _init_login_manager(app) 50 | _configure_logger() 51 | init_scheduler(app.config.get("SQLALCHEMY_DATABASE_URI")) 52 | db.init(app.config.get("SQLALCHEMY_DATABASE_URI")) 53 | 54 | 55 | def _configure_app(app_): 56 | """Configure the Flask WSGI app.""" 57 | app_.url_map.strict_slashes = False 58 | app_.config.from_object(default_settings) 59 | app_.config.from_envvar("JOB_CONFIG", silent=True) 60 | db_url = app_.config.get("SQLALCHEMY_DATABASE_URI") 61 | if not db_url: 62 | raise Exception("No db_url in config") 63 | app_.wsgi_app = ProxyFix(app_.wsgi_app) 64 | 65 | global SSL_VERIFY 66 | if app_.config.get("SSL_VERIFY") in ["False", "FALSE", "0", False, 0]: 67 | SSL_VERIFY = False 68 | else: 69 | SSL_VERIFY = True 70 | 71 | return app_ 72 | 73 | 74 | def _init_login_manager(app_): 75 | """Initialise and configure the login manager.""" 76 | login_manager = flogin.LoginManager() 77 | login_manager.setup_app(app_) 78 | login_manager.anonymous_user = Anonymous 79 | login_manager.login_view = "login" 80 | 81 | users = {app_.config["USERNAME"]: User("Admin", 0)} 82 | names = dict((int(v.get_id()), k) for k, v in list(users.items())) 83 | 84 | @login_manager.user_loader 85 | def load_user(userid): 86 | userid = int(userid) 87 | name = names.get(userid) 88 | return users.get(name) 89 | 90 | return users, names 91 | 92 | 93 | def _configure_logger_for_production(logger): 94 | """Configure the given logger for production deployment. 95 | 96 | Logs to stderr and file, and emails errors to admins. 97 | 98 | """ 99 | stderr_handler = logging.StreamHandler(sys.stderr) 100 | stderr_handler.setLevel(logging.INFO) 101 | ts_formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s") 102 | if "STDERR" in app.config and app.config["STDERR"]: 103 | stderr_handler.setFormatter(ts_formatter) 104 | logger.addHandler(stderr_handler) 105 | 106 | if "LOG_FILE" in app.config and app.config["LOG_FILE"]: 107 | file_handler = logging.handlers.RotatingFileHandler( 108 | app.config.get("LOG_FILE"), maxBytes=67108864, backupCount=5 109 | ) 110 | file_handler.setLevel(logging.INFO) 111 | file_handler.setFormatter(ts_formatter) 112 | logger.addHandler(file_handler) 113 | 114 | mail_handler = logging.handlers.SMTPHandler( 115 | "127.0.0.1", 116 | app.config.get("FROM_EMAIL"), 117 | app.config.get("ADMINS", []), 118 | "CKAN Service Error", 119 | ) 120 | mail_handler.setLevel(logging.ERROR) 121 | if "FROM_EMAIL" in app.config: 122 | logger.addHandler(mail_handler) 123 | 124 | 125 | def _configure_logger_for_debugging(logger): 126 | """Configure the given logger for debug mode.""" 127 | logger.addHandler(app.logger.handlers[0]) 128 | 129 | 130 | def _configure_logger(): 131 | """Configure the logging module.""" 132 | if not app.debug: 133 | _configure_logger_for_production(logging.getLogger()) 134 | elif not app.testing: 135 | _configure_logger_for_debugging(logging.getLogger()) 136 | 137 | 138 | def init_scheduler(db_uri): 139 | """Initialise and configure the scheduler.""" 140 | global scheduler 141 | scheduler = Scheduler() 142 | scheduler.misfire_grace_time = 3600 143 | scheduler.add_jobstore(sqlalchemy_store.SQLAlchemyJobStore(url=db_uri), "default") 144 | scheduler.add_listener( 145 | job_listener, 146 | events.EVENT_JOB_EXECUTED | events.EVENT_JOB_MISSED | events.EVENT_JOB_ERROR, 147 | ) 148 | return scheduler 149 | 150 | 151 | class User(flogin.UserMixin): 152 | def __init__(self, name, id, active=True): 153 | self.name = name 154 | self.id = id 155 | self.active = active 156 | 157 | def is_active(self): 158 | return self.active 159 | 160 | 161 | class Anonymous(flogin.AnonymousUserMixin): 162 | name = "Anonymous" 163 | 164 | 165 | class RunNowTrigger(object): 166 | """Custom apscheduler trigger to run job once and only 167 | once""" 168 | 169 | def __init__(self): 170 | self.run = False 171 | 172 | def get_next_fire_time(self, start_date): 173 | if not self.run: 174 | self.run = True 175 | return datetime.datetime.now() 176 | 177 | def __str__(self): 178 | return "RunTriggerNow, run = %s" % self.run 179 | 180 | def __repr__(self): 181 | return "RunTriggerNow, run = %s" % self.run 182 | 183 | 184 | def job_listener(event): 185 | """Listens to completed job""" 186 | aps_job_id = event.job_id 187 | 188 | job = db.get_job(aps_job_id, use_aps_id=True) 189 | job_id = job['job_id'] 190 | 191 | if event.code == events.EVENT_JOB_MISSED: 192 | db.mark_job_as_missed(job_id) 193 | elif event.exception: 194 | if isinstance(event.exception, util.JobError): 195 | error_object = event.exception.as_dict() 196 | else: 197 | error_object = "\n".join( 198 | [event.traceback] + [repr(event.exception)] 199 | ) 200 | db.mark_job_as_errored(job_id, error_object) 201 | else: 202 | db.mark_job_as_completed(job_id, event.retval) 203 | api_key = db.get_job(job_id)["api_key"] 204 | result_ok = send_result(job_id, api_key) 205 | 206 | if not result_ok: 207 | db.mark_job_as_failed_to_post_result(job_id) 208 | 209 | # Optionally notify tests that job_listener() has finished. 210 | if "_TEST_CALLBACK_URL" in app.config: 211 | requests.get(app.config["_TEST_CALLBACK_URL"]) 212 | 213 | 214 | headers = {str("Content-Type"): str("application/json")} 215 | 216 | 217 | @app.route("/", methods=["GET"]) 218 | def index(): 219 | """Show link to documentation. 220 | 221 | :rtype: A dictionary with the following keys 222 | :param help: Help text 223 | :type help: string 224 | """ 225 | return flask.jsonify( 226 | help=""" 227 | Get help at: 228 | http://ckan-service-provider.readthedocs.org/.""" 229 | ) 230 | 231 | 232 | @app.route("/status", methods=["GET"]) 233 | def status(): 234 | """Show version, available job types and name of service. 235 | 236 | **Results:** 237 | 238 | :rtype: A dictionary with the following keys 239 | :param version: Version of the service provider 240 | :type version: float 241 | :param job_types: Available job types 242 | :type job_types: list of strings 243 | :param name: Name of the service 244 | :type name: string 245 | :param stats: Shows stats for jobs in queue 246 | :type stats: dictionary 247 | """ 248 | job_types = sorted(list(async_types.keys()) + list(sync_types.keys())) 249 | 250 | counts = {} 251 | for job_status in job_statuses: 252 | counts[job_status] = db.ENGINE.execute( 253 | db.JOBS_TABLE.count().where(db.JOBS_TABLE.c.status == job_status) 254 | ).first()[0] 255 | 256 | return flask.jsonify( 257 | version=0.1, 258 | job_types=job_types, 259 | name=app.config.get("NAME", "example"), 260 | stats=counts, 261 | ) 262 | 263 | 264 | def check_auth(username, password): 265 | """This function is called to check if a username / 266 | password combination is valid. 267 | """ 268 | return username == app.config["USERNAME"] and password == app.config["PASSWORD"] 269 | 270 | 271 | @app.route("/login", methods=["POST", "GET"]) 272 | def login(): 273 | """Log in as administrator 274 | 275 | You can use wither basic auth or form based login (via POST). 276 | 277 | :param username: The administrator's username 278 | :type username: string 279 | :param password: The administrator's password 280 | :type password: string 281 | """ 282 | username = None 283 | password = None 284 | next = flask.request.args.get("next") 285 | auth = flask.request.authorization 286 | 287 | if flask.request.method == "POST": 288 | username = flask.request.form["username"] 289 | password = flask.request.form["password"] 290 | 291 | if auth and auth.type == "basic": 292 | username = auth.username 293 | password = auth.password 294 | 295 | if not flogin.current_user.is_active: 296 | error = "You have to login with proper credentials" 297 | if username and password: 298 | if check_auth(username, password): 299 | user = _users.get(username) 300 | if user: 301 | if flogin.login_user(user): 302 | return flask.redirect(next or flask.url_for("user")) 303 | error = "Could not log in user." 304 | else: 305 | error = "User not found." 306 | else: 307 | error = "Wrong username or password." 308 | else: 309 | error = "No username or password." 310 | return flask.Response( 311 | "Could not verify your access level for that URL.\n {}".format(error), 312 | 401, 313 | {str("WWW-Authenticate"): str('Basic realm="Login Required"')}, 314 | ) 315 | return flask.redirect(next or flask.url_for("user")) 316 | 317 | 318 | @app.route("/user", methods=["GET"]) 319 | def user(): 320 | """Show information about the current user 321 | 322 | :rtype: A dictionary with the following keys 323 | :param id: User id 324 | :type id: int 325 | :param name: User name 326 | :type name: string 327 | :param is_active: Whether the user is currently active 328 | :type is_active: bool 329 | :param is_anonymous: The anonymous user is the default user if you 330 | are not logged in 331 | :type is_anonymous: bool 332 | """ 333 | user = flogin.current_user 334 | return flask.jsonify( 335 | { 336 | "id": user.get_id(), 337 | "name": user.name, 338 | "is_active": user.is_active(), 339 | "is_anonymous": user.is_anonymous, 340 | } 341 | ) 342 | 343 | 344 | @app.route("/logout") 345 | def logout(): 346 | """Log out the active user""" 347 | flogin.logout_user() 348 | next = flask.request.args.get("next") 349 | return flask.redirect(next or flask.url_for("user")) 350 | 351 | 352 | @app.route("/job", methods=["GET"]) 353 | def job_list(): 354 | """List all jobs. 355 | 356 | :param _limit: maximum number of jobs to show (default 100) 357 | :type _limit: int 358 | :param _offset: how many jobs to skip before showin the first one (default 0) 359 | :type _offset: int 360 | :param _status: filter jobs by status (complete, error) 361 | :type _status: string 362 | 363 | Also, you can filter the jobs by their metadata. Use the metadata key 364 | as parameter key and the value as value. 365 | 366 | :rtype: A list of job ids 367 | """ 368 | args = dict((key, value) for key, value in list(flask.request.args.items())) 369 | limit = args.pop("_limit", 100) 370 | offset = args.pop("_offset", 0) 371 | 372 | select = ( 373 | sql.select( 374 | [db.JOBS_TABLE.c.job_id], 375 | from_obj=[ 376 | db.JOBS_TABLE.outerjoin( 377 | db.METADATA_TABLE, 378 | db.JOBS_TABLE.c.job_id == db.METADATA_TABLE.c.job_id, 379 | ) 380 | ], 381 | ) 382 | .group_by(db.JOBS_TABLE.c.job_id) 383 | .order_by(db.JOBS_TABLE.c.requested_timestamp.desc()) 384 | .limit(limit) 385 | .offset(offset) 386 | ) 387 | 388 | status = args.pop("_status", None) 389 | if status: 390 | select = select.where(db.JOBS_TABLE.c.status == status) 391 | 392 | ors = [] 393 | for key, value in args.items(): 394 | # Turn strings into unicode to stop SQLAlchemy 395 | # "Unicode type received non-unicode bind param value" warnings. 396 | key = str(key) 397 | 398 | ors.append( 399 | sql.and_(db.METADATA_TABLE.c.key == key, db.METADATA_TABLE.c.value == value) 400 | ) 401 | 402 | if ors: 403 | select = select.where(sql.or_(*ors)) 404 | select = select.having(sql.func.count(db.JOBS_TABLE.c.job_id) == len(ors)) 405 | 406 | result = db.ENGINE.execute(select) 407 | listing = [] 408 | for (job_id,) in result: 409 | listing.append(flask.url_for("job_status", job_id=job_id)) 410 | 411 | return flask.jsonify(list=listing) 412 | 413 | 414 | class DatetimeJsonEncoder(json.JSONEncoder): 415 | # Custon JSON encoder 416 | def default(self, obj): 417 | if isinstance(obj, datetime.datetime): 418 | return obj.isoformat() 419 | 420 | return json.JSONEncoder.default(self, obj) 421 | 422 | 423 | @app.route("/job/", methods=["GET"]) 424 | def job_status(job_id, show_job_key=False, ignore_auth=False): 425 | """Show a specific job. 426 | 427 | :param limit: Limit the number of logs 428 | :type limit: integer 429 | 430 | **Results:** 431 | 432 | :rtype: A dictionary with the following keys 433 | :param status: Status of job (complete, error) 434 | :type status: string 435 | :param sent_data: Input data for job 436 | :type sent_data: json encodable data 437 | :param job_id: An identifier for the job 438 | :type job_id: string 439 | :param result_url: Callback url 440 | :type result_url: url string 441 | :param data: Results from job. 442 | :type data: json encodable data 443 | :param error: Error raised during job execution 444 | :type error: string 445 | :param metadata: Metadata provided when submitting job. 446 | :type metadata: list of key - value pairs 447 | :param requested_timestamp: Time the job started 448 | :type requested_timestamp: timestamp 449 | :param finished_timestamp: Time the job finished 450 | :type finished_timestamp: timestamp 451 | 452 | :statuscode 200: no error 453 | :statuscode 403: not authorized to view the job's data 454 | :statuscode 404: job id not found 455 | :statuscode 409: an error occurred 456 | """ 457 | limit = flask.request.args.get("limit") 458 | job_dict = db.get_job(job_id, limit=limit) 459 | if not job_dict: 460 | return json.dumps({"error": "job_id not found"}), 404, headers 461 | if not ignore_auth and not is_authorized(job_dict): 462 | return json.dumps({"error": "not authorized"}), 403, headers 463 | job_dict.pop("api_key", None) 464 | if not show_job_key: 465 | job_dict.pop("job_key", None) 466 | return flask.Response( 467 | json.dumps(job_dict, cls=DatetimeJsonEncoder), mimetype="application/json" 468 | ) 469 | 470 | 471 | @app.route("/job/", methods=["DELETE"]) 472 | def job_delete(job_id): 473 | """Deletes the job together with its logs and metadata. 474 | 475 | :param job_id: An identifier for the job 476 | :type job_id: string 477 | 478 | :statuscode 200: no error 479 | :statuscode 403: not authorized to delete the job 480 | :statuscode 404: the job could not be found 481 | :statuscode 409: an error occurred 482 | """ 483 | conn = db.ENGINE.connect() 484 | job = db.get_job(job_id) 485 | if not job: 486 | return json.dumps({"error": "job_id not found"}), 404, headers 487 | if not is_authorized(job): 488 | return json.dumps({"error": "not authorized"}), 403, headers 489 | trans = conn.begin() 490 | try: 491 | conn.execute(db.JOBS_TABLE.delete().where(db.JOBS_TABLE.c.job_id == job_id)) 492 | trans.commit() 493 | return json.dumps({"success": True}), 200, headers 494 | except Exception as e: 495 | trans.rollback() 496 | return json.dumps({"error": str(e)}), 409, headers 497 | finally: 498 | conn.close() 499 | 500 | 501 | @app.route("/job", methods=["DELETE"]) 502 | def clear_jobs(): 503 | """Clear old jobs 504 | 505 | :param days: Jobs for how many days should be kept (default: 10) 506 | :type days: integer 507 | 508 | :statuscode 200: no error 509 | :statuscode 403: not authorized to delete jobs 510 | :statuscode 409: an error occurred 511 | """ 512 | if not is_authorized(): 513 | return json.dumps({"error": "not authorized"}), 403, headers 514 | 515 | days = flask.request.args.get("days", None) 516 | return _clear_jobs(days) 517 | 518 | 519 | def _clear_jobs(days=None): 520 | if days is None: 521 | days = app.config.get("KEEP_JOBS_AGE") 522 | else: 523 | try: 524 | days = int(days) 525 | except Exception as e: 526 | return json.dumps({"error": str(e)}), 409, headers 527 | conn = db.ENGINE.connect() 528 | trans = conn.begin() 529 | date = datetime.datetime.now() - datetime.timedelta(days=days) 530 | try: 531 | conn.execute( 532 | db.JOBS_TABLE.delete().where(db.JOBS_TABLE.c.finished_timestamp < date) 533 | ) 534 | trans.commit() 535 | return json.dumps({"success": True}), 200, headers 536 | except Exception as e: 537 | trans.rollback() 538 | return json.dumps({"error": str(e)}), 409, headers 539 | finally: 540 | conn.close() 541 | 542 | 543 | @app.route("/job//data", methods=["GET"]) 544 | def job_data(job_id): 545 | """Get the raw data that the job returned. The mimetype 546 | will be the value provided in the metdata for the key ``mimetype``. 547 | 548 | **Results:** 549 | 550 | :rtype: string 551 | 552 | :statuscode 200: no error 553 | :statuscode 403: not authorized to view the job's data 554 | :statuscode 404: job id not found 555 | :statuscode 409: an error occurred 556 | """ 557 | job_dict = db.get_job(job_id) 558 | if not job_dict: 559 | return json.dumps({"error": "job_id not found"}), 404, headers 560 | if not is_authorized(job_dict): 561 | return json.dumps({"error": "not authorized"}), 403, headers 562 | if job_dict["error"]: 563 | return json.dumps({"error": job_dict["error"]}), 409, headers 564 | content_type = job_dict["metadata"].get("mimetype") 565 | return flask.Response(job_dict["data"], mimetype=content_type) 566 | 567 | 568 | @app.route("/job/", methods=["POST"]) 569 | @app.route("/job", methods=["POST"]) 570 | def job(job_id=None): 571 | """Submit a job. If no id is provided, a random id will be generated. 572 | 573 | :param job_type: Which kind of job should be run. Has to be one of the 574 | available job types. 575 | :type job_type: string 576 | :param api_key: An API key that is needed to execute the job. This could 577 | be a CKAN API key that is needed to write any data. The key will also be 578 | used to administer jobs. If you don't want to use a real API key, you can 579 | provide a random string that you keep secure. 580 | :type api_key: string 581 | :param data: Data that is send to the job as input. (Optional) 582 | :type data: json encodable data 583 | :param result_url: Callback url that is called once the job has finished. 584 | (Optional) 585 | :type result_url: url string 586 | :param metadata: Data needed for the execution of the job which is not 587 | the input data. (Optional) 588 | :type metadata: list of key - value pairs 589 | 590 | **Results:** 591 | 592 | :rtype: A dictionary with the following keys 593 | :param job_id: An identifier for the job 594 | :type job_id: string 595 | :param job_key: A key that is required to view and administer the job 596 | :type job_key: string 597 | 598 | :statuscode 200: no error 599 | :statuscode 409: an error occurred 600 | 601 | """ 602 | if not job_id: 603 | job_id = str(uuid.uuid4()) 604 | 605 | # key required for job administration 606 | job_key = str(uuid.uuid4()) 607 | 608 | ############# ERROR CHECKING ################ 609 | try: 610 | input = flask.request.json 611 | except werkzeug.exceptions.BadRequest: 612 | return json.dumps({"error": "Malformed json"}), 409, headers 613 | 614 | # Idk why but this is needed for some libraries that 615 | # send malformed content types 616 | content_type = flask.request.content_type or "" 617 | if not input and "application/json" in content_type.lower(): 618 | try: 619 | input = json.loads(flask.request.data) 620 | except ValueError: 621 | pass 622 | if not input: 623 | return ( 624 | json.dumps( 625 | { 626 | "error": ( 627 | "Not recognised as json, make " 628 | "sure content type is application/" 629 | "json" 630 | ) 631 | } 632 | ), 633 | 409, 634 | headers, 635 | ) 636 | 637 | ACCEPTED_ARGUMENTS = set( 638 | ["job_type", "data", "metadata", "result_url", "api_key", "metadata"] 639 | ) 640 | extra_keys = set(input.keys()) - ACCEPTED_ARGUMENTS 641 | if extra_keys: 642 | return ( 643 | json.dumps( 644 | { 645 | "error": ( 646 | "Too many arguments. Extra keys are {}".format( 647 | ", ".join(extra_keys) 648 | ) 649 | ) 650 | } 651 | ), 652 | 409, 653 | headers, 654 | ) 655 | 656 | # check result_url here as good to give warning early. 657 | result_url = input.get("result_url") 658 | if result_url and not result_url.startswith("http"): 659 | return json.dumps({"error": "result_url has to start with http"}), 409, headers 660 | 661 | job_type = input.get("job_type") 662 | if not job_type: 663 | return json.dumps({"error": "Please specify a job type"}), 409, headers 664 | 665 | job_types = list(async_types.keys()) + list(sync_types.keys()) 666 | 667 | if job_type not in job_types: 668 | error_string = ("Job type {} not available. Available job types are {}").format( 669 | job_type, ", ".join(sorted(job_types)) 670 | ) 671 | return json.dumps({"error": error_string}), 409, headers 672 | 673 | api_key = input.get("api_key") 674 | if not api_key: 675 | return json.dumps({"error": "Please provide your API key."}), 409, headers 676 | 677 | metadata = input.get("metadata", {}) 678 | if not isinstance(metadata, dict): 679 | return json.dumps({"error": "metadata has to be a json object"}), 409, headers 680 | ############# END CHECKING ################ 681 | 682 | synchronous_job = sync_types.get(job_type) 683 | if synchronous_job: 684 | return run_synchronous_job(synchronous_job, job_id, job_key, input) 685 | else: 686 | asynchronous_job = async_types.get(job_type) 687 | return run_asynchronous_job(asynchronous_job, job_id, job_key, input) 688 | 689 | 690 | def run_synchronous_job(job, job_id, job_key, input): 691 | try: 692 | db.add_pending_job(job_id, job_key, **input) 693 | except sa.exc.IntegrityError as e: 694 | error_string = "job_id {} already exists".format(job_id) 695 | return json.dumps({"error": error_string}), 409, headers 696 | 697 | try: 698 | result = job(job_id, input) 699 | 700 | if hasattr(result, "__call__"): 701 | db.mark_job_as_completed(job_id) 702 | return flask.Response(result(), mimetype="application/json") 703 | else: 704 | db.mark_job_as_completed(job_id, result) 705 | 706 | except util.JobError as e: 707 | db.mark_job_as_errored(job_id, e.as_dict()) 708 | except Exception as e: 709 | db.mark_job_as_errored( 710 | job_id, traceback.format_tb(sys.exc_info()[2])[-1] + repr(e) 711 | ) 712 | 713 | api_key = db.get_job(job_id)["api_key"] 714 | result_ok = send_result(job_id, api_key) 715 | 716 | if not result_ok: 717 | db.mark_job_as_failed_to_post_result(job_id) 718 | 719 | return job_status(job_id=job_id, show_job_key=True, ignore_auth=True) 720 | 721 | 722 | def run_asynchronous_job(job, job_id, job_key, input): 723 | if not scheduler.running: 724 | scheduler.start() 725 | try: 726 | db.add_pending_job(job_id, job_key, **input) 727 | except sa.exc.IntegrityError: 728 | error_string = "job_id {} already exists".format(job_id) 729 | return json.dumps({"error": error_string}), 409, headers 730 | trigger = DateTrigger() 731 | aps_job = scheduler.add_job(job, trigger, [job_id, input], None) 732 | 733 | db.set_aps_job_id(job_id, aps_job.id) 734 | 735 | return job_status(job_id=job_id, show_job_key=True, ignore_auth=True) 736 | 737 | 738 | def is_authorized(job=None): 739 | """Returns true if the request is authorized for the job 740 | if provided. If no job is provided, the user has to be admin 741 | to be authorized. 742 | """ 743 | if flogin.current_user.is_authenticated: 744 | return True 745 | if job: 746 | job_key = flask.request.headers.get("Authorization") 747 | if job_key == app.config.get("SECRET_KEY"): 748 | return True 749 | return job["job_key"] == job_key 750 | return False 751 | 752 | 753 | def send_result(job_id, api_key=None): 754 | """Send results to where requested. 755 | 756 | If api_key is provided, it is used, otherwiese 757 | the key from the job will be used. 758 | """ 759 | job_dict = db.get_job(job_id) 760 | result_url = job_dict.get("result_url") 761 | 762 | if not result_url: 763 | 764 | # A job with an API key (for using when posting to the callback URL) 765 | # but no callback URL is weird, but it can happen. 766 | db.delete_api_key(job_id) 767 | 768 | return True 769 | 770 | api_key_from_job = job_dict.pop("api_key", None) 771 | if not api_key: 772 | api_key = api_key_from_job 773 | headers = {"Content-Type": "application/json"} 774 | if api_key: 775 | if ":" in api_key: 776 | header, key = api_key.split(":") 777 | else: 778 | header, key = "Authorization", api_key 779 | headers[header] = key 780 | 781 | try: 782 | result = requests.post( 783 | result_url, 784 | data=json.dumps(job_dict, cls=DatetimeJsonEncoder), 785 | headers=headers, 786 | verify=SSL_VERIFY, 787 | ) 788 | 789 | db.delete_api_key(job_id) 790 | 791 | except requests.ConnectionError: 792 | return False 793 | 794 | return result.status_code == requests.codes.ok 795 | 796 | 797 | def main(): 798 | init() 799 | app.run(app.config.get("HOST"), app.config.get("PORT")) 800 | 801 | 802 | if __name__ == "__main__": 803 | main() 804 | -------------------------------------------------------------------------------- /doc/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | PAPER = 8 | BUILDDIR = _build 9 | 10 | # Internal variables. 11 | PAPEROPT_a4 = -D latex_paper_size=a4 12 | PAPEROPT_letter = -D latex_paper_size=letter 13 | ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 14 | # the i18n builder cannot share the environment and doctrees with the others 15 | I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . 16 | 17 | .PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext 18 | 19 | help: 20 | @echo "Please use \`make ' where is one of" 21 | @echo " html to make standalone HTML files" 22 | @echo " dirhtml to make HTML files named index.html in directories" 23 | @echo " singlehtml to make a single large HTML file" 24 | @echo " pickle to make pickle files" 25 | @echo " json to make JSON files" 26 | @echo " htmlhelp to make HTML files and a HTML help project" 27 | @echo " qthelp to make HTML files and a qthelp project" 28 | @echo " devhelp to make HTML files and a Devhelp project" 29 | @echo " epub to make an epub" 30 | @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" 31 | @echo " latexpdf to make LaTeX files and run them through pdflatex" 32 | @echo " text to make text files" 33 | @echo " man to make manual pages" 34 | @echo " texinfo to make Texinfo files" 35 | @echo " info to make Texinfo files and run them through makeinfo" 36 | @echo " gettext to make PO message catalogs" 37 | @echo " changes to make an overview of all changed/added/deprecated items" 38 | @echo " linkcheck to check all external links for integrity" 39 | @echo " doctest to run all doctests embedded in the documentation (if enabled)" 40 | 41 | clean: 42 | -rm -rf $(BUILDDIR)/* 43 | 44 | html: 45 | $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html 46 | @echo 47 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." 48 | 49 | dirhtml: 50 | $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml 51 | @echo 52 | @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." 53 | 54 | singlehtml: 55 | $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml 56 | @echo 57 | @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." 58 | 59 | pickle: 60 | $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle 61 | @echo 62 | @echo "Build finished; now you can process the pickle files." 63 | 64 | json: 65 | $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json 66 | @echo 67 | @echo "Build finished; now you can process the JSON files." 68 | 69 | htmlhelp: 70 | $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp 71 | @echo 72 | @echo "Build finished; now you can run HTML Help Workshop with the" \ 73 | ".hhp project file in $(BUILDDIR)/htmlhelp." 74 | 75 | qthelp: 76 | $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp 77 | @echo 78 | @echo "Build finished; now you can run "qcollectiongenerator" with the" \ 79 | ".qhcp project file in $(BUILDDIR)/qthelp, like this:" 80 | @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/messytables.qhcp" 81 | @echo "To view the help file:" 82 | @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/messytables.qhc" 83 | 84 | devhelp: 85 | $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp 86 | @echo 87 | @echo "Build finished." 88 | @echo "To view the help file:" 89 | @echo "# mkdir -p $$HOME/.local/share/devhelp/messytables" 90 | @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/messytables" 91 | @echo "# devhelp" 92 | 93 | epub: 94 | $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub 95 | @echo 96 | @echo "Build finished. The epub file is in $(BUILDDIR)/epub." 97 | 98 | latex: 99 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 100 | @echo 101 | @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." 102 | @echo "Run \`make' in that directory to run these through (pdf)latex" \ 103 | "(use \`make latexpdf' here to do that automatically)." 104 | 105 | latexpdf: 106 | $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex 107 | @echo "Running LaTeX files through pdflatex..." 108 | make -C $(BUILDDIR)/latex all-pdf 109 | @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." 110 | 111 | text: 112 | $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text 113 | @echo 114 | @echo "Build finished. The text files are in $(BUILDDIR)/text." 115 | 116 | man: 117 | $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man 118 | @echo 119 | @echo "Build finished. The manual pages are in $(BUILDDIR)/man." 120 | 121 | texinfo: 122 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 123 | @echo 124 | @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." 125 | @echo "Run \`make' in that directory to run these through makeinfo" \ 126 | "(use \`make info' here to do that automatically)." 127 | 128 | info: 129 | $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo 130 | @echo "Running Texinfo files through makeinfo..." 131 | make -C $(BUILDDIR)/texinfo info 132 | @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." 133 | 134 | gettext: 135 | $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale 136 | @echo 137 | @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." 138 | 139 | changes: 140 | $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes 141 | @echo 142 | @echo "The overview file is in $(BUILDDIR)/changes." 143 | 144 | linkcheck: 145 | $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck 146 | @echo 147 | @echo "Link check complete; look for any errors in the above output " \ 148 | "or in $(BUILDDIR)/linkcheck/output.txt." 149 | 150 | doctest: 151 | $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest 152 | @echo "Testing of doctests in the sources finished, look at the " \ 153 | "results in $(BUILDDIR)/doctest/output.txt." 154 | -------------------------------------------------------------------------------- /doc/_themes/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010 by Armin Ronacher. 2 | 3 | Some rights reserved. 4 | 5 | Redistribution and use in source and binary forms of the theme, with or 6 | without modification, are permitted provided that the following conditions 7 | are met: 8 | 9 | * Redistributions of source code must retain the above copyright 10 | notice, this list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above 13 | copyright notice, this list of conditions and the following 14 | disclaimer in the documentation and/or other materials provided 15 | with the distribution. 16 | 17 | * The names of the contributors may not be used to endorse or 18 | promote products derived from this software without specific 19 | prior written permission. 20 | 21 | We kindly ask you to only use these themes in an unmodified manner just 22 | for Flask and Flask-related products, not for unrelated projects. If you 23 | like the visual style and want to use it for your own projects, please 24 | consider making some larger changes to the themes (such as changing 25 | font faces, sizes, colors or margins). 26 | 27 | THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 28 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 29 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 30 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 31 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 | ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE 37 | POSSIBILITY OF SUCH DAMAGE. 38 | -------------------------------------------------------------------------------- /doc/_themes/flask_small/layout.html: -------------------------------------------------------------------------------- 1 | {% extends "basic/layout.html" %} 2 | {% block header %} 3 | {{ super() }} 4 | {% if pagename == 'index' %} 5 |
6 | {% endif %} 7 | {% endblock %} 8 | {% block footer %} 9 | {% if pagename == 'index' %} 10 |
11 | {% endif %} 12 | {% endblock %} 13 | {# do not display relbars #} 14 | {% block relbar1 %}{% endblock %} 15 | {% block relbar2 %} 16 | {% if theme_github_fork %} 17 | Fork me on GitHub 19 | {% endif %} 20 | {% endblock %} 21 | {% block sidebar1 %}{% endblock %} 22 | {% block sidebar2 %}{% endblock %} 23 | -------------------------------------------------------------------------------- /doc/_themes/flask_small/static/flasky.css_t: -------------------------------------------------------------------------------- 1 | /* 2 | * flasky.css_t 3 | * ~~~~~~~~~~~~ 4 | * 5 | * Sphinx stylesheet -- flasky theme based on nature theme. 6 | * 7 | * :copyright: Copyright 2007-2010 by the Sphinx team, see AUTHORS. 8 | * :license: BSD, see LICENSE for details. 9 | * 10 | */ 11 | 12 | @import url("basic.css"); 13 | 14 | /* -- page layout ----------------------------------------------------------- */ 15 | 16 | body { 17 | font-family: 'Georgia', serif; 18 | font-size: 17px; 19 | color: #000; 20 | background: white; 21 | margin: 0; 22 | padding: 0; 23 | } 24 | 25 | div.documentwrapper { 26 | float: left; 27 | width: 100%; 28 | } 29 | 30 | div.bodywrapper { 31 | margin: 40px auto 0 auto; 32 | width: 700px; 33 | } 34 | 35 | hr { 36 | border: 1px solid #B1B4B6; 37 | } 38 | 39 | div.body { 40 | background-color: #ffffff; 41 | color: #3E4349; 42 | padding: 0 30px 30px 30px; 43 | } 44 | 45 | img.floatingflask { 46 | padding: 0 0 10px 10px; 47 | float: right; 48 | } 49 | 50 | div.footer { 51 | text-align: right; 52 | color: #888; 53 | padding: 10px; 54 | font-size: 14px; 55 | width: 650px; 56 | margin: 0 auto 40px auto; 57 | } 58 | 59 | div.footer a { 60 | color: #888; 61 | text-decoration: underline; 62 | } 63 | 64 | div.related { 65 | line-height: 32px; 66 | color: #888; 67 | } 68 | 69 | div.related ul { 70 | padding: 0 0 0 10px; 71 | } 72 | 73 | div.related a { 74 | color: #444; 75 | } 76 | 77 | /* -- body styles ----------------------------------------------------------- */ 78 | 79 | a { 80 | color: #004B6B; 81 | text-decoration: underline; 82 | } 83 | 84 | a:hover { 85 | color: #6D4100; 86 | text-decoration: underline; 87 | } 88 | 89 | div.body { 90 | padding-bottom: 40px; /* saved for footer */ 91 | } 92 | 93 | div.body h1, 94 | div.body h2, 95 | div.body h3, 96 | div.body h4, 97 | div.body h5, 98 | div.body h6 { 99 | font-family: 'Garamond', 'Georgia', serif; 100 | font-weight: normal; 101 | margin: 30px 0px 10px 0px; 102 | padding: 0; 103 | } 104 | 105 | {% if theme_index_logo %} 106 | /*div.indexwrapper h1 { 107 | text-indent: -999999px; 108 | background: url({{ theme_index_logo }}) no-repeat center center; 109 | height: {{ theme_index_logo_height }}; 110 | }*/ 111 | {% endif %} 112 | 113 | div.body h2 { font-size: 180%; } 114 | div.body h3 { font-size: 150%; } 115 | div.body h4 { font-size: 130%; } 116 | div.body h5 { font-size: 100%; } 117 | div.body h6 { font-size: 100%; } 118 | 119 | a.headerlink { 120 | color: white; 121 | padding: 0 4px; 122 | text-decoration: none; 123 | } 124 | 125 | a.headerlink:hover { 126 | color: #444; 127 | background: #eaeaea; 128 | } 129 | 130 | div.body p, div.body dd, div.body li { 131 | line-height: 1.4em; 132 | } 133 | 134 | div.admonition { 135 | background: #fafafa; 136 | margin: 20px -30px; 137 | padding: 10px 30px; 138 | border-top: 1px solid #ccc; 139 | border-bottom: 1px solid #ccc; 140 | } 141 | 142 | div.admonition p.admonition-title { 143 | font-family: 'Garamond', 'Georgia', serif; 144 | font-weight: normal; 145 | font-size: 24px; 146 | margin: 0 0 10px 0; 147 | padding: 0; 148 | line-height: 1; 149 | } 150 | 151 | div.admonition p.last { 152 | margin-bottom: 0; 153 | } 154 | 155 | div.highlight{ 156 | background-color: white; 157 | } 158 | 159 | dt:target, .highlight { 160 | background: #FAF3E8; 161 | } 162 | 163 | div.note { 164 | background-color: #eee; 165 | border: 1px solid #ccc; 166 | } 167 | 168 | div.seealso { 169 | background-color: #ffc; 170 | border: 1px solid #ff6; 171 | } 172 | 173 | div.topic { 174 | background-color: #eee; 175 | } 176 | 177 | div.warning { 178 | background-color: #ffe4e4; 179 | border: 1px solid #f66; 180 | } 181 | 182 | p.admonition-title { 183 | display: inline; 184 | } 185 | 186 | p.admonition-title:after { 187 | content: ":"; 188 | } 189 | 190 | pre, tt { 191 | font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; 192 | font-size: 0.85em; 193 | } 194 | 195 | img.screenshot { 196 | } 197 | 198 | tt.descname, tt.descclassname { 199 | font-size: 0.95em; 200 | } 201 | 202 | tt.descname { 203 | padding-right: 0.08em; 204 | } 205 | 206 | img.screenshot { 207 | -moz-box-shadow: 2px 2px 4px #eee; 208 | -webkit-box-shadow: 2px 2px 4px #eee; 209 | box-shadow: 2px 2px 4px #eee; 210 | } 211 | 212 | table.docutils { 213 | border: 1px solid #888; 214 | -moz-box-shadow: 2px 2px 4px #eee; 215 | -webkit-box-shadow: 2px 2px 4px #eee; 216 | box-shadow: 2px 2px 4px #eee; 217 | } 218 | 219 | table.docutils td, table.docutils th { 220 | border: 1px solid #888; 221 | padding: 0.25em 0.7em; 222 | } 223 | 224 | table.field-list, table.footnote { 225 | border: none; 226 | -moz-box-shadow: none; 227 | -webkit-box-shadow: none; 228 | box-shadow: none; 229 | } 230 | 231 | table.footnote { 232 | margin: 15px 0; 233 | width: 100%; 234 | border: 1px solid #eee; 235 | } 236 | 237 | table.field-list th { 238 | padding: 0 0.8em 0 0; 239 | } 240 | 241 | table.field-list td { 242 | padding: 0; 243 | } 244 | 245 | table.footnote td { 246 | padding: 0.5em; 247 | } 248 | 249 | dl { 250 | margin: 0; 251 | padding: 0; 252 | } 253 | 254 | dl dd { 255 | margin-left: 30px; 256 | } 257 | 258 | pre { 259 | padding: 0; 260 | margin: 15px -30px; 261 | padding: 8px; 262 | line-height: 1.3em; 263 | padding: 7px 30px; 264 | background: #eee; 265 | border-radius: 2px; 266 | -moz-border-radius: 2px; 267 | -webkit-border-radius: 2px; 268 | } 269 | 270 | dl pre { 271 | margin-left: -60px; 272 | padding-left: 60px; 273 | } 274 | 275 | tt { 276 | background-color: #ecf0f3; 277 | color: #222; 278 | /* padding: 1px 2px; */ 279 | } 280 | 281 | tt.xref, a tt { 282 | background-color: #FBFBFB; 283 | } 284 | 285 | a:hover tt { 286 | background: #EEE; 287 | } 288 | -------------------------------------------------------------------------------- /doc/_themes/flask_small/theme.conf: -------------------------------------------------------------------------------- 1 | [theme] 2 | inherit = basic 3 | stylesheet = flasky.css 4 | nosidebar = true 5 | pygments_style = flask_theme_support.FlaskyStyle 6 | 7 | [options] 8 | index_logo = '' 9 | index_logo_height = 120px 10 | github_fork = okfn/ckan-service-provider 11 | -------------------------------------------------------------------------------- /doc/_themes/flask_theme_support.py: -------------------------------------------------------------------------------- 1 | # flasky extensions. flasky pygments style based on tango style 2 | from pygments.style import Style 3 | from pygments.token import ( 4 | Keyword, 5 | Name, 6 | Comment, 7 | String, 8 | Error, 9 | Number, 10 | Operator, 11 | Generic, 12 | Whitespace, 13 | Punctuation, 14 | Other, 15 | Literal, 16 | ) 17 | 18 | 19 | class FlaskyStyle(Style): 20 | background_color = "#f8f8f8" 21 | default_style = "" 22 | 23 | styles = { 24 | # No corresponding class for the following: 25 | # Text: "", # class: '' 26 | Whitespace: "underline #f8f8f8", # class: 'w' 27 | Error: "#a40000 border:#ef2929", # class: 'err' 28 | Other: "#000000", # class 'x' 29 | Comment: "italic #8f5902", # class: 'c' 30 | Comment.Preproc: "noitalic", # class: 'cp' 31 | Keyword: "bold #004461", # class: 'k' 32 | Keyword.Constant: "bold #004461", # class: 'kc' 33 | Keyword.Declaration: "bold #004461", # class: 'kd' 34 | Keyword.Namespace: "bold #004461", # class: 'kn' 35 | Keyword.Pseudo: "bold #004461", # class: 'kp' 36 | Keyword.Reserved: "bold #004461", # class: 'kr' 37 | Keyword.Type: "bold #004461", # class: 'kt' 38 | Operator: "#582800", # class: 'o' 39 | Operator.Word: "bold #004461", # class: 'ow' - like keywords 40 | Punctuation: "bold #000000", # class: 'p' 41 | # because special names such as Name.Class, Name.Function, etc. 42 | # are not recognized as such later in the parsing, we choose them 43 | # to look the same as ordinary variables. 44 | Name: "#000000", # class: 'n' 45 | Name.Attribute: "#c4a000", # class: 'na' - to be revised 46 | Name.Builtin: "#004461", # class: 'nb' 47 | Name.Builtin.Pseudo: "#3465a4", # class: 'bp' 48 | Name.Class: "#000000", # class: 'nc' - to be revised 49 | Name.Constant: "#000000", # class: 'no' - to be revised 50 | Name.Decorator: "#888", # class: 'nd' - to be revised 51 | Name.Entity: "#ce5c00", # class: 'ni' 52 | Name.Exception: "bold #cc0000", # class: 'ne' 53 | Name.Function: "#000000", # class: 'nf' 54 | Name.Property: "#000000", # class: 'py' 55 | Name.Label: "#f57900", # class: 'nl' 56 | Name.Namespace: "#000000", # class: 'nn' - to be revised 57 | Name.Other: "#000000", # class: 'nx' 58 | Name.Tag: "bold #004461", # class: 'nt' - like a keyword 59 | Name.Variable: "#000000", # class: 'nv' - to be revised 60 | Name.Variable.Class: "#000000", # class: 'vc' - to be revised 61 | Name.Variable.Global: "#000000", # class: 'vg' - to be revised 62 | Name.Variable.Instance: "#000000", # class: 'vi' - to be revised 63 | Number: "#990000", # class: 'm' 64 | Literal: "#000000", # class: 'l' 65 | Literal.Date: "#000000", # class: 'ld' 66 | String: "#4e9a06", # class: 's' 67 | String.Backtick: "#4e9a06", # class: 'sb' 68 | String.Char: "#4e9a06", # class: 'sc' 69 | String.Doc: "italic #8f5902", # class: 'sd' - like a comment 70 | String.Double: "#4e9a06", # class: 's2' 71 | String.Escape: "#4e9a06", # class: 'se' 72 | String.Heredoc: "#4e9a06", # class: 'sh' 73 | String.Interpol: "#4e9a06", # class: 'si' 74 | String.Other: "#4e9a06", # class: 'sx' 75 | String.Regex: "#4e9a06", # class: 'sr' 76 | String.Single: "#4e9a06", # class: 's1' 77 | String.Symbol: "#4e9a06", # class: 'ss' 78 | Generic: "#000000", # class: 'g' 79 | Generic.Deleted: "#a40000", # class: 'gd' 80 | Generic.Emph: "italic #000000", # class: 'ge' 81 | Generic.Error: "#ef2929", # class: 'gr' 82 | Generic.Heading: "bold #000080", # class: 'gh' 83 | Generic.Inserted: "#00A000", # class: 'gi' 84 | Generic.Output: "#888", # class: 'go' 85 | Generic.Prompt: "#745334", # class: 'gp' 86 | Generic.Strong: "bold #000000", # class: 'gs' 87 | Generic.Subheading: "bold #800080", # class: 'gu' 88 | Generic.Traceback: "bold #a40000", # class: 'gt' 89 | } 90 | -------------------------------------------------------------------------------- /doc/conf.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # messytables documentation build configuration file, created by 4 | # sphinx-quickstart on Sun Aug 14 17:09:50 2011. 5 | # 6 | # This file is execfile()d with the current directory set to its containing dir. 7 | # 8 | # Note that not all possible configuration values are present in this 9 | # autogenerated file. 10 | # 11 | # All configuration values have a default; values that are commented out 12 | # serve to show the default. 13 | 14 | import sys 15 | import os 16 | 17 | # If extensions (or modules to document with autodoc) are in another directory, 18 | # add these directories to sys.path here. If the directory is relative to the 19 | # documentation root, use os.path.abspath to make it absolute, like shown here. 20 | # sys.path.insert(0, os.path.abspath('.')) 21 | 22 | # -- General configuration ----------------------------------------------------- 23 | 24 | # If your documentation needs a minimal Sphinx version, state it here. 25 | # needs_sphinx = '1.0' 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be extensions 28 | # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 29 | extensions = ["sphinx.ext.autodoc", "sphinxcontrib.autohttp.flask"] 30 | 31 | # Add any paths that contain templates here, relative to this directory. 32 | templates_path = ["_templates"] 33 | 34 | # The suffix of source filenames. 35 | source_suffix = ".rst" 36 | 37 | # The encoding of source files. 38 | # source_encoding = 'utf-8-sig' 39 | 40 | # The master toctree document. 41 | master_doc = "index" 42 | 43 | # General information about the project. 44 | project = "ckanserviceprovider" 45 | copyright = "2013, Open Knowledge Foundation" 46 | 47 | # The version info for the project you're documenting, acts as replacement for 48 | # |version| and |release|, also used in various other places throughout the 49 | # built documents. 50 | # 51 | # The short X.Y version. 52 | version = "0.1" 53 | # The full version, including alpha/beta/rc tags. 54 | release = "0.1" 55 | 56 | # The language for content autogenerated by Sphinx. Refer to documentation 57 | # for a list of supported languages. 58 | # language = None 59 | 60 | # There are two options for replacing |today|: either, you set today to some 61 | # non-false value, then it is used: 62 | # today = '' 63 | # Else, today_fmt is used as the format for a strftime call. 64 | # today_fmt = '%B %d, %Y' 65 | 66 | # List of patterns, relative to source directory, that match files and 67 | # directories to ignore when looking for source files. 68 | exclude_patterns = ["_build"] 69 | 70 | # The reST default role (used for this markup: `text`) to use for all documents. 71 | # default_role = None 72 | 73 | # If true, '()' will be appended to :func: etc. cross-reference text. 74 | # add_function_parentheses = True 75 | 76 | # If true, the current module name will be prepended to all description 77 | # unit titles (such as .. function::). 78 | # add_module_names = True 79 | 80 | # If true, sectionauthor and moduleauthor directives will be shown in the 81 | # output. They are ignored by default. 82 | # show_authors = False 83 | 84 | # The name of the Pygments (syntax highlighting) style to use. 85 | pygments_style = "sphinx" 86 | 87 | # A list of ignored prefixes for module index sorting. 88 | # modindex_common_prefix = [] 89 | 90 | # -- Options for HTML output --------------------------------------------------- 91 | 92 | # The theme to use for HTML and HTML Help pages. See the documentation for 93 | # a list of builtin themes. 94 | sys.path.append(os.path.abspath("_themes")) 95 | html_theme_path = ["_themes"] 96 | html_theme = "flask_small" 97 | 98 | # Theme options are theme-specific and customize the look and feel of a theme 99 | # further. For a list of options available for each theme, see the 100 | # documentation. 101 | # html_theme_options = {} 102 | 103 | # Add any paths that contain custom themes here, relative to this directory. 104 | # html_theme_path = [] 105 | 106 | # The name for this set of Sphinx documents. If None, it defaults to 107 | # " v documentation". 108 | # html_title = None 109 | 110 | # A shorter title for the navigation bar. Default is the same as html_title. 111 | # html_short_title = None 112 | 113 | # The name of an image file (relative to this directory) to place at the top 114 | # of the sidebar. 115 | # html_logo = None 116 | 117 | # The name of an image file (within the static path) to use as favicon of the 118 | # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 119 | # pixels large. 120 | # html_favicon = None 121 | 122 | # Add any paths that contain custom static files (such as style sheets) here, 123 | # relative to this directory. They are copied after the builtin static files, 124 | # so a file named "default.css" will overwrite the builtin "default.css". 125 | html_static_path = ["_static"] 126 | 127 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, 128 | # using the given strftime format. 129 | # html_last_updated_fmt = '%b %d, %Y' 130 | 131 | # If true, SmartyPants will be used to convert quotes and dashes to 132 | # typographically correct entities. 133 | # html_use_smartypants = True 134 | 135 | # Custom sidebar templates, maps document names to template names. 136 | # html_sidebars = {} 137 | 138 | # Additional templates that should be rendered to pages, maps page names to 139 | # template names. 140 | # html_additional_pages = {} 141 | 142 | # If false, no module index is generated. 143 | # html_domain_indices = True 144 | 145 | # If false, no index is generated. 146 | # html_use_index = True 147 | 148 | # If true, the index is split into individual pages for each letter. 149 | # html_split_index = False 150 | 151 | # If true, links to the reST sources are added to the pages. 152 | # html_show_sourcelink = True 153 | 154 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 155 | # html_show_sphinx = True 156 | 157 | # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. 158 | # html_show_copyright = True 159 | 160 | # If true, an OpenSearch description file will be output, and all pages will 161 | # contain a tag referring to it. The value of this option must be the 162 | # base URL from which the finished HTML is served. 163 | # html_use_opensearch = '' 164 | 165 | # This is the file name suffix for HTML files (e.g. ".xhtml"). 166 | # html_file_suffix = None 167 | 168 | # Output file base name for HTML help builder. 169 | htmlhelp_basename = "ckanservicedoc" 170 | 171 | 172 | # -- Options for LaTeX output -------------------------------------------------- 173 | 174 | latex_elements = { 175 | # The paper size ('letterpaper' or 'a4paper'). 176 | #'papersize': 'letterpaper', 177 | # The font size ('10pt', '11pt' or '12pt'). 178 | #'pointsize': '10pt', 179 | # Additional stuff for the LaTeX preamble. 180 | #'preamble': '', 181 | } 182 | 183 | # Grouping the document tree into LaTeX files. List of tuples 184 | # (source start file, target name, title, author, documentclass [howto/manual]). 185 | latex_documents = [ 186 | ( 187 | "index", 188 | "ckanservice.tex", 189 | "CKAN Service Provider Documentation", 190 | "Open Knowledge Foundation", 191 | "manual", 192 | ), 193 | ] 194 | 195 | # The name of an image file (relative to this directory) to place at the top of 196 | # the title page. 197 | # latex_logo = None 198 | 199 | # For "manual" documents, if this is true, then toplevel headings are parts, 200 | # not chapters. 201 | # latex_use_parts = False 202 | 203 | # If true, show page references after internal links. 204 | # latex_show_pagerefs = False 205 | 206 | # If true, show URL addresses after external links. 207 | # latex_show_urls = False 208 | 209 | # Documents to append as an appendix to all manuals. 210 | # latex_appendices = [] 211 | 212 | # If false, no module index is generated. 213 | # latex_domain_indices = True 214 | 215 | 216 | # -- Options for manual page output -------------------------------------------- 217 | 218 | # One entry per manual page. List of tuples 219 | # (source start file, name, description, authors, manual section). 220 | man_pages = [ 221 | ( 222 | "index", 223 | "ckanservice", 224 | "CKAN Service Provider Documentation", 225 | ["Open Knowledge Foundation"], 226 | 1, 227 | ) 228 | ] 229 | 230 | # If true, show URL addresses after external links. 231 | # man_show_urls = False 232 | 233 | 234 | # -- Options for Texinfo output ------------------------------------------------ 235 | 236 | # Grouping the document tree into Texinfo files. List of tuples 237 | # (source start file, target name, title, author, 238 | # dir menu entry, description, category) 239 | texinfo_documents = [ 240 | ( 241 | "index", 242 | "ckanservice", 243 | "CKAN Service Provider Documentation", 244 | "Open Knowledge Foundation", 245 | "messytables", 246 | "One line description of project.", 247 | "Miscellaneous", 248 | ), 249 | ] 250 | 251 | # Documents to append as an appendix to all manuals. 252 | # texinfo_appendices = [] 253 | 254 | # If false, no module index is generated. 255 | # texinfo_domain_indices = True 256 | 257 | # How to display URL addresses: 'footnote', 'no', or 'inline'. 258 | # texinfo_show_urls = 'footnote' 259 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | CKAN Service Provider 2 | ===================== 3 | 4 | A simple flask app that makes functions available as synchronous or asynchronous jobs. 5 | 6 | Routes 7 | ------ 8 | 9 | .. autoflask:: ckanserviceprovider.web:app 10 | :undoc-static: 11 | :include-empty-docstring: 12 | 13 | Administration 14 | -------------- 15 | 16 | To view the results of a job or resubmit it, the job key, that is returned when a job is created, 17 | is needed. Alternatively, you can log in as admin or provide the secure key. The credentials for 18 | the admin user and the secure key stored in the settings file. 19 | 20 | Add a job 21 | --------- 22 | 23 | Just decorate your function and it will become available as a job:: 24 | 25 | import ckanserviceprovider.job as job 26 | import ckanserviceprovider.util as util 27 | 28 | @job.synchronous 29 | def echo(task_id, input): 30 | handler = util.StoringHandler(task_id, input) 31 | logger = logging.getLogger(__name__) 32 | logger.addHandler(handler) 33 | 34 | if input['data'].startswith('>'): 35 | raise util.JobError('do not start message with >') 36 | if input['data'].startswith('#'): 37 | raise Exception('serious exception') 38 | if input['data'].startswith('&'): 39 | logger.warn('just a warning') 40 | return '>' + input['data'] 41 | 42 | Expected job errors should be raised as `util.JobError`. For logging, use the handler 43 | ``util.StoringHandler`` to make sure that the logs are properly saved. 44 | 45 | -------------------------------------------------------------------------------- /example/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ckan/ckan-service-provider/3f7fc99a9fbed5bbac064b3068887aff97adfcc3/example/__init__.py -------------------------------------------------------------------------------- /example/jobs.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import ckanserviceprovider.job as job 4 | import ckanserviceprovider.util as util 5 | 6 | 7 | @job.synchronous 8 | def echo(task_id, input): 9 | if input["data"].startswith(">"): 10 | raise util.JobError("do not start message with >") 11 | if input["data"].startswith("#"): 12 | raise Exception("serious exception") 13 | return ">" + input["data"] 14 | 15 | 16 | @job.asynchronous 17 | def async_echo(task_id, input): 18 | if input["data"].startswith(">"): 19 | raise util.JobError("do not start message with >") 20 | if input["data"].startswith("#"): 21 | raise Exception("serious exception") 22 | return ">" + input["data"] 23 | 24 | 25 | @job.asynchronous 26 | def async_ping(task_id, input): 27 | handler = util.StoringHandler(task_id, input) 28 | logger = logging.getLogger(__name__) 29 | logger.addHandler(handler) 30 | 31 | logger.warn("ping") 32 | return "ping" 33 | -------------------------------------------------------------------------------- /example/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import ckanserviceprovider.web as web 4 | 5 | import jobs 6 | 7 | # check whether jobs have been imported properly 8 | assert jobs.echo 9 | 10 | 11 | def serve(): 12 | web.init() 13 | web.app.run(web.app.config.get("HOST"), web.app.config.get("PORT")) 14 | 15 | 16 | def serve_test(): 17 | web.init() 18 | return web.app.test_client() 19 | 20 | 21 | def main(): 22 | import argparse 23 | 24 | argparser = argparse.ArgumentParser( 25 | description="Example service", 26 | epilog='''"For a moment, nothing happened. 27 | Then, after a second or so, nothing continued to happen."''', 28 | ) 29 | 30 | argparser.add_argument( 31 | "config", 32 | metavar="CONFIG", 33 | type=argparse.FileType("r"), 34 | help="configuration file", 35 | ) 36 | args = argparser.parse_args() 37 | 38 | os.environ["JOB_CONFIG"] = os.path.abspath(args.config.name) 39 | serve() 40 | 41 | 42 | if __name__ == "__main__": 43 | main() 44 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | sphinxcontrib-httpdomain>=1.1.7 2 | httpretty==1.1.4 3 | pytest 4 | pytest-cov 5 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Suggested versions 2 | APScheduler==3.9.1.post1 3 | requests==2.27.1 4 | Flask==2.1.1 5 | Flask-Login==0.5.0 6 | future==0.18.3 7 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [egg_info] 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages # Always prefer setuptools over distutils 2 | from codecs import open # To use a consistent encoding 3 | from os import path 4 | import sys 5 | 6 | 7 | here = path.abspath(path.dirname(__file__)) 8 | 9 | # Get the long description from the relevant file 10 | with open(path.join(here, "README.md"), encoding="utf-8") as f: 11 | long_description = f.read() 12 | 13 | install_requires = [ 14 | "APScheduler>=2.1.2,<3.10.0", 15 | "Flask>=1.1.1,<3.0.0", 16 | "Werkzeug>=1.0.0,<3.0.0", 17 | "SQLAlchemy>=1.3.15,<1.4.0", 18 | "requests>=2.23.0", 19 | "future", 20 | ] 21 | 22 | if sys.version_info[1] <= 6: 23 | install_requires.append("flask-login==0.5.0") 24 | else: 25 | install_requires.append("flask-login==0.6.2") 26 | 27 | 28 | setup( 29 | name="ckanserviceprovider", 30 | # Versions should comply with PEP440. For a discussion on single-sourcing 31 | # the version across setup.py and the project code, see 32 | # http://packaging.python.org/en/latest/tutorial.html#version 33 | version="1.2.0", 34 | description="A library for making web services that make functions available as synchronous or asynchronous jobs", 35 | long_description=long_description, 36 | long_description_content_type="text/markdown", 37 | # The project's main homepage. 38 | url="https://github.com/ckan/ckan-service-provider", 39 | # Choose your license 40 | license="AGPL", 41 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers 42 | classifiers=[ 43 | # How mature is this project? Common values are 44 | # 3 - Alpha 45 | # 4 - Beta 46 | # 5 - Production/Stable 47 | "Development Status :: 5 - Production/Stable", 48 | # Pick your license as you wish (should match "license" above) 49 | "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)", 50 | # Specify the Python versions you support here. In particular, ensure 51 | # that you indicate whether you support Python 2, Python 3 or both. 52 | "Programming Language :: Python :: 3.7", 53 | "Programming Language :: Python :: 3.8", 54 | "Programming Language :: Python :: 3.9", 55 | "Programming Language :: Python :: 3.10", 56 | "Programming Language :: Python :: 3.11", 57 | ], 58 | # What does your project relate to? 59 | keywords="ckan", 60 | # You can just specify the packages manually here if your project is 61 | # simple. Or you can use find_packages(). 62 | packages=find_packages(exclude=["doc", "tests*"]), 63 | # List run-time dependencies here. These will be installed by pip when your 64 | # project is installed. For an analysis of "install_requires" vs pip's 65 | # requirements files see: 66 | # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files 67 | install_requires=install_requires, 68 | # If there are data files included in your packages that need to be 69 | # installed, specify them here. If using Python 2.6 or less, then these 70 | # have to be included in MANIFEST.in as well. 71 | package_data={}, 72 | # Although 'package_data' is the preferred approach, in some case you may 73 | # need to place data files outside of your packages. 74 | # see http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files 75 | # In this case, 'data_file' will be installed into '/my_data' 76 | data_files=[], 77 | # To provide executable scripts, use entry points in preference to the 78 | # "scripts" keyword. Entry points provide cross-platform support and allow 79 | # pip to create the appropriate form of executable for the target platform. 80 | entry_points={}, 81 | ) 82 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ckan/ckan-service-provider/3f7fc99a9fbed5bbac064b3068887aff97adfcc3/tests/__init__.py -------------------------------------------------------------------------------- /tests/settings_test.py: -------------------------------------------------------------------------------- 1 | DEBUG = True 2 | TESTING = True 3 | USERNAME = "testadmin" 4 | PASSWORD = "testpass" 5 | SQLALCHEMY_DATABASE_URI = "sqlite:////tmp/job_store.db" 6 | NAME = "testing" 7 | _TEST_CALLBACK_URL = "http://0.0.0.0:8721/test_callback_url" 8 | -------------------------------------------------------------------------------- /tests/test_db.py: -------------------------------------------------------------------------------- 1 | """Unit tests for ckanserviceprovider/db.py.""" 2 | import pytest 3 | 4 | import ckanserviceprovider.db as db 5 | 6 | 7 | def test_validate_error_with_none(): 8 | """_validate_error() should return None if given None.""" 9 | assert db._validate_error(None) is None 10 | 11 | 12 | def test_validate_error_with_string(): 13 | """If given a string _validate_error() should return it wrapped in a dict.""" 14 | assert db._validate_error("Something went wrong") == { 15 | "message": "Something went wrong" 16 | } 17 | 18 | 19 | def test_validate_error_with_valid_dict(): 20 | """If given a valid dict _validate_error() should return the same dict.""" 21 | job_dict = {"message": "Something went wrong"} 22 | assert db._validate_error(job_dict) == job_dict 23 | 24 | 25 | def test_validate_error_with_dict_with_invalid_error(): 26 | """_validate_error() should raise if given a dict with an invalid message.""" 27 | job_dict = {"message": 42} # Error message is invalid: it's not a string. 28 | with pytest.raises(db.InvalidErrorObjectError): 29 | db._validate_error(job_dict) 30 | 31 | 32 | def test_validate_error_with_dict_with_no_error_key(): 33 | """_validate_error() should raise if given a dict with no "message" key.""" 34 | job_dict = {"foo": "bar"} 35 | with pytest.raises(db.InvalidErrorObjectError): 36 | db._validate_error(job_dict) 37 | 38 | 39 | def test_validate_error_with_random_object(): 40 | """_validate_error() should raise if given an object of the wrong type.""" 41 | 42 | class Foo(object): 43 | pass 44 | 45 | # An error object that is not None and is not string- or dict-like at all. 46 | error_obj = Foo() 47 | 48 | with pytest.raises(db.InvalidErrorObjectError): 49 | db._validate_error(error_obj) 50 | -------------------------------------------------------------------------------- /tests/test_joberror.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | 4 | from ckanserviceprovider.util import JobError 5 | 6 | 7 | def test_joberror(): 8 | err = JobError("oh no ❌") 9 | assert err.message == "oh no ❌" 10 | -------------------------------------------------------------------------------- /tests/test_web.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import json 4 | import time 5 | import logging 6 | import uuid 7 | import threading 8 | 9 | import httpretty 10 | import pytest 11 | 12 | import ckanserviceprovider.web as web 13 | import ckanserviceprovider.job as job 14 | import ckanserviceprovider.util as util 15 | import ckanserviceprovider.db as db 16 | 17 | 18 | # The callback URL that ckanserviceprovider will post to when the 19 | # asynchronous background job finishes. We will mock this. 20 | RESULT_URL = "http://0.0.0.0/ckanserviceprovider/result_url" 21 | 22 | 23 | def configure(): 24 | """Configure the Flask app. 25 | 26 | This has to be called just once per test run (not e.g. once for each test). 27 | 28 | """ 29 | os.environ["JOB_CONFIG"] = os.path.join( 30 | os.path.dirname(__file__), "settings_test.py" 31 | ) 32 | web.init() 33 | 34 | 35 | configure() 36 | 37 | 38 | def reset_db(): 39 | """Reset the database and scheduler. 40 | 41 | Should be called after each test. 42 | 43 | """ 44 | db.drop_all() 45 | db.init(web.app.config.get("SQLALCHEMY_DATABASE_URI")) 46 | web.init_scheduler(web.app.config.get("SQLALCHEMY_DATABASE_URI")) 47 | 48 | 49 | def test_client(): 50 | """Return a test client for the ckanserviceprovider web app.""" 51 | return web.app.test_client() 52 | 53 | 54 | def login(client, username="testadmin", password="testpass"): 55 | return client.post( 56 | "/login", data=dict(username=username, password=password), follow_redirects=True 57 | ) 58 | 59 | 60 | def _make_request_callback_function(event): 61 | """Return an httpretty request callback function that sets the given event. 62 | 63 | This is a helper function for mock_result_url() below. 64 | 65 | """ 66 | 67 | def request_callback(request, uri, headers): 68 | event.set() 69 | return (200, headers, "") 70 | 71 | return request_callback 72 | 73 | 74 | def mock_result_url(result_url): 75 | """Mock the given CKAN Service Provider result URL. 76 | 77 | Returns a threading.Event object that you can use to wait for the mock URL 78 | to be called by doing: event.wait(). 79 | 80 | The way it works is: 81 | 82 | * A test method calls this function to mock a result_url, receives a 83 | threading event object in return. 84 | 85 | * The test method posts to ckanserviceprovider passing the mocked 86 | result_url. 87 | 88 | * ckanserviceprovider kicks off an asynchronous background job. 89 | 90 | * The test method waits for ckanserviceprovider's asynchronous background 91 | job to finish by doing event.wait(). 92 | 93 | * When the job finishes ckanserviceprovider posts to the result_url. 94 | 95 | * The post is intercepted and redirected to a function that sets the 96 | thread event. 97 | 98 | * event.wait() returns and the test method continues. 99 | 100 | """ 101 | event = threading.Event() 102 | request_callback = _make_request_callback_function(event) 103 | httpretty.register_uri(httpretty.POST, result_url, body=request_callback) 104 | return event 105 | 106 | 107 | def _mock_test_callback_url(client): 108 | """Mock the tests callback URL. 109 | 110 | _TEST_CALLBACK_URL is a special URL that CKAN service provider calls after 111 | it has completely finished with an asynchronous job. Waiting for this URL 112 | to be called enables tests to assert things that don't happen until after 113 | the normal client callback URL has been called, without any race conditions 114 | in the tests. 115 | 116 | Returns a threading.Event object that you can use to wait for the mock URL 117 | to be called by doing: event.wait(). 118 | 119 | """ 120 | test_callback_url = client.application.config.get("_TEST_CALLBACK_URL") 121 | event = threading.Event() 122 | request_callback = _make_request_callback_function(event) 123 | httpretty.register_uri(httpretty.GET, test_callback_url, body=request_callback) 124 | return event 125 | 126 | 127 | def number_of_jobs(client): 128 | """Return the number of jobs that the app has in its database. 129 | 130 | :param client: a test client of the ckanserviceprovider flask app 131 | 132 | """ 133 | return len(json.loads(client.get("/job").data)["list"]) 134 | 135 | 136 | @job.synchronous 137 | def echo(task_id, input_): 138 | if input_["data"].startswith(">"): 139 | raise util.JobError("Do not start message with >") 140 | if input_["data"].startswith("#"): 141 | raise Exception("Something went totally wrong") 142 | return ">" + input_["data"] 143 | 144 | 145 | @job.synchronous 146 | def echo_raw(task_id, input_): 147 | if input_["data"].startswith(">"): 148 | raise util.JobError("Do not start message with >") 149 | 150 | def raw(): 151 | for x in sorted(input_["data"]): 152 | yield x 153 | 154 | return raw 155 | 156 | 157 | @job.asynchronous 158 | def example(task_id, input_): 159 | if "time" not in input_["data"]: 160 | raise util.JobError("time not in input") 161 | 162 | time.sleep(input_["data"]["time"]) 163 | return "Slept for " + str(input_["data"]["time"]) + " seconds." 164 | 165 | 166 | @job.asynchronous 167 | def failing(task_id, input_): 168 | time.sleep(0.1) 169 | raise util.JobError("failed") 170 | 171 | 172 | @job.asynchronous 173 | def log(task_id, input_): 174 | handler = util.StoringHandler(task_id, input_) 175 | logger = logging.Logger(task_id) 176 | logger.addHandler(handler) 177 | 178 | logger.warning("Just a warning") 179 | 180 | 181 | class TestWeb(object): 182 | def teardown(self): 183 | reset_db() 184 | 185 | def test_get_job_id_with_limit(self): 186 | """Limiting logs works fine""" 187 | client = test_client() 188 | client.post( 189 | "/job/12345", 190 | data=json.dumps( 191 | {"job_type": "example", "api_key": 42, "data": {"time": 0.1}} 192 | ), 193 | content_type="application/json", 194 | ) 195 | db.add_logs(job_id="12345", message="message1") 196 | db.add_logs(job_id="12345", message="message2") 197 | db.add_logs(job_id="12345", message="message3") 198 | 199 | # Make sure it works without limit 200 | response = client.get( 201 | "/job/12345", headers={"Authorization": "please_replace_me"} 202 | ) 203 | return_data = json.loads(response.data) 204 | assert len(return_data["logs"]) == 3 205 | 206 | # Now test with limit 207 | response = client.get( 208 | "/job/12345?limit=2", headers={"Authorization": "please_replace_me"} 209 | ) 210 | return_data = json.loads(response.data) 211 | assert len(return_data["logs"]) == 2 212 | 213 | def test_status(self): 214 | """/status should return JSON with the app version, job types, etc.""" 215 | client = test_client() 216 | response = client.get("/status") 217 | status_data = json.loads(response.data) 218 | status_data.pop("stats") 219 | assert status_data == dict( 220 | version=0.1, 221 | job_types=sorted(["failing", "example", "log", "echo_raw", "echo"]), 222 | name="testing", 223 | ) 224 | 225 | def test_content_type(self): 226 | """Pages should have content_type "application/json".""" 227 | client = test_client() 228 | for page in ["/job", "/status", "/job/foo"]: 229 | response = client.get(page) 230 | assert response.content_type == "application/json" 231 | 232 | def test_bad_post(self): 233 | """Invalid posts to /job should receive error messages in JSON.""" 234 | client = test_client() 235 | response = client.post("/job", data='{"ffsfsafsa":"moo"}') 236 | try: 237 | assert json.loads(response.data) == { 238 | "error": "Not recognised as json, make sure content type is " 239 | "application/json" 240 | } 241 | except AssertionError: 242 | assert json.loads(response.data) == {"error": "Malformed json"} 243 | except json.JSONDecodeError: 244 | assert response.status_code == 415 245 | 246 | response = client.post( 247 | "/job", data='{"ffsfsafsa":moo}', content_type="application/json" 248 | ) 249 | assert json.loads(response.data) == {"error": "Malformed json"} 250 | 251 | response = client.post( 252 | "/job", 253 | data=json.dumps({"api_key": 42, "data": {"time": 5}}), 254 | content_type="application/json", 255 | ) 256 | assert json.loads(response.data) == {"error": "Please specify a job type"} 257 | 258 | response = client.post( 259 | "/job", 260 | data=json.dumps({"job_type": "moo", "api_key": 42, "data": {"time": 5}}), 261 | content_type="application/json", 262 | ) 263 | assert json.loads(response.data) == { 264 | "error": "Job type moo not available. Available job types are " 265 | "echo, echo_raw, example, failing, log" 266 | } 267 | 268 | response = client.post( 269 | "/job", 270 | data=json.dumps({"job_type": "example", "data": {"time": 5}}), 271 | content_type="application/json", 272 | ) 273 | assert json.loads(response.data) == {"error": "Please provide your API key."} 274 | 275 | response = client.post( 276 | "/job", 277 | data=json.dumps( 278 | {"job_type": "example", "api_key": 42, "data": {"time": 5}, "foo": 42} 279 | ), 280 | content_type="application/json", 281 | ) 282 | assert json.loads(response.data) == { 283 | "error": "Too many arguments. Extra keys are foo" 284 | } 285 | 286 | def test_asynchronous_post_with_good_job(self): 287 | """A valid post to /job should get back a JSON object with a job ID.""" 288 | client = test_client() 289 | response = client.post( 290 | "/job", 291 | data=json.dumps( 292 | {"job_type": "example", "api_key": 42, "data": {"time": 0.1}} 293 | ), 294 | content_type="application/json", 295 | ) 296 | 297 | return_data = json.loads(response.data) 298 | assert "job_id" in return_data 299 | 300 | @httpretty.activate 301 | def test_callback_url_is_called_with_api_key(self): 302 | """It should use the API key when posting to the callback URL.""" 303 | API_KEY = "42" 304 | client = test_client() 305 | event = threading.Event() 306 | 307 | def callback(request, uri, headers): 308 | assert request.headers.get("Authorization") == API_KEY, ( 309 | "ckanserviceprovider should put the API key in the " 310 | "Authorization header when calling the callback URL" 311 | ) 312 | event.set() 313 | return (200, headers, "") 314 | 315 | httpretty.register_uri(httpretty.POST, RESULT_URL, body=callback) 316 | 317 | client.post( 318 | "/job", 319 | data=json.dumps( 320 | { 321 | "job_type": "example", 322 | "api_key": API_KEY, 323 | "data": {"time": 0.1}, 324 | "result_url": RESULT_URL, 325 | } 326 | ), 327 | content_type="application/json", 328 | ) 329 | 330 | timeout = 10.0 331 | assert event.wait( 332 | timeout 333 | ), "result_url was not called within {timeout} seconds".format(timeout=timeout) 334 | 335 | @httpretty.activate 336 | def test_get_job_does_not_return_api_key(self): 337 | """The dict that get_job() returns should not contain the API key.""" 338 | client = test_client() 339 | mock_result_url(RESULT_URL) 340 | event = _mock_test_callback_url(client) 341 | 342 | response = client.post( 343 | "/job", 344 | data=json.dumps( 345 | { 346 | "job_type": "example", 347 | "api_key": 42, 348 | "data": {"time": 0.1}, 349 | "result_url": RESULT_URL, 350 | } 351 | ), 352 | content_type="application/json", 353 | ) 354 | return_data = json.loads(response.data) 355 | 356 | timeout = 10.0 357 | assert event.wait( 358 | timeout 359 | ), "_TEST_CALLBACK_URL was not called within {timeout} " "seconds".format( 360 | timeout=timeout 361 | ) 362 | 363 | job_ = db.get_job(return_data["job_id"]) 364 | assert not job_["api_key"], job_ 365 | 366 | def test_post_job_with_custom_id(self): 367 | """Posting a job with a custom ID should return the ID in the JSON.""" 368 | client = test_client() 369 | 370 | response = client.post( 371 | "/job/moo", 372 | data=json.dumps( 373 | {"job_type": "example", "api_key": 42, "data": {"time": 0.1}} 374 | ), 375 | content_type="application/json", 376 | ) 377 | 378 | assert json.loads(response.data)["job_id"] == "moo", json.loads(response.data) 379 | 380 | # FIXME: I think there's actually a race condition here - if the 381 | # asynchronous background job (running in another thread) finishes before 382 | # we get to the assert it'l fail. 383 | def test_get_job_while_pending(self): 384 | """Create a job with a custom ID and get the job while still pending. 385 | 386 | This tests the value of the job's metadata while the job is still in a 387 | pending status. 388 | 389 | """ 390 | client = test_client() 391 | client.post( 392 | "/job/moo", 393 | data=json.dumps( 394 | {"job_type": "example", "api_key": 42, "data": {"time": 1}} 395 | ), 396 | content_type="application/json", 397 | ) 398 | 399 | login(client) 400 | response = client.get("/job/moo") 401 | 402 | job_status_data = json.loads(response.data) 403 | job_status_data.pop("requested_timestamp") 404 | 405 | job_status_data.pop("aps_job_id") 406 | assert job_status_data == { 407 | "status": "pending", 408 | "sent_data": {"time": 1}, 409 | "job_id": "moo", 410 | "finished_timestamp": None, 411 | "job_type": "example", 412 | "error": None, 413 | "data": None, 414 | "metadata": {}, 415 | "logs": [], 416 | "result_url": None, 417 | }, job_status_data 418 | 419 | @httpretty.activate 420 | def test_get_job_when_completed(self): 421 | """Get a job with a custom ID after it has completed. 422 | 423 | Tests the value of the job's metadata after the job has completed. 424 | 425 | """ 426 | client = test_client() 427 | event = mock_result_url(RESULT_URL) 428 | client.post( 429 | "/job/moo", 430 | data=json.dumps( 431 | { 432 | "job_type": "example", 433 | "api_key": 42, 434 | "data": {"time": 0.1}, 435 | "result_url": RESULT_URL, 436 | } 437 | ), 438 | content_type="application/json", 439 | ) 440 | 441 | timeout = 10.0 442 | assert event.wait( 443 | timeout 444 | ), "result_url was not called within {timeout} seconds".format(timeout=timeout) 445 | 446 | login(client) 447 | 448 | response = client.get("/job/moo") 449 | 450 | job_status_data = json.loads(response.data) 451 | job_status_data.pop("requested_timestamp") 452 | job_status_data.pop("finished_timestamp") 453 | job_status_data.pop("aps_job_id") 454 | 455 | assert job_status_data == { 456 | "status": "complete", 457 | "sent_data": {"time": 0.1}, 458 | "job_id": "moo", 459 | "job_type": "example", 460 | "error": None, 461 | "data": "Slept for 0.1 seconds.", 462 | "metadata": {}, 463 | "logs": [], 464 | "result_url": RESULT_URL, 465 | }, job_status_data 466 | 467 | def test_post_job_with_duplicate_custom_id(self): 468 | """Posting a job with a duplicate ID should error.""" 469 | client = test_client() 470 | client.post( 471 | "/job/moo", 472 | data=json.dumps( 473 | {"job_type": "example", "api_key": 42, "data": {"time": 0.1}} 474 | ), 475 | content_type="application/json", 476 | ) 477 | 478 | response = client.post( 479 | "/job/moo", 480 | data=json.dumps( 481 | {"job_type": "example", "api_key": 42, "data": {"time": 0.1}} 482 | ), 483 | content_type="application/json", 484 | ) 485 | 486 | assert json.loads(response.data) == { 487 | "error": "job_id moo already " "exists" 488 | }, json.loads(response.data) 489 | 490 | def test_post_with_job_error(self): 491 | """If a job raises JobError the response should still contain job_id. 492 | 493 | If a job with a custom ID raises JobError then the "job_id" field in 494 | ckanserviceprovider's HTTP response should still contain the job's 495 | custom ID. 496 | 497 | """ 498 | # The 'example' job type (defined above) will raise JobError for this 499 | # data because the data has no "time" key. 500 | client = test_client() 501 | response = client.post( 502 | "/job/missing_time", 503 | data=json.dumps({"job_type": "example", "api_key": 42, "data": {}}), 504 | content_type="application/json", 505 | ) 506 | 507 | assert json.loads(response.data)["job_id"] == "missing_time", json.loads( 508 | response.data 509 | ) 510 | 511 | def test_post_with_job_exception(self): 512 | """If a job raises an exception the HTTP response should have an error. 513 | 514 | If a job raises an arbitrary exception (e.g. because of a mistake in 515 | the job code) the response to the job post HTTP request should have 516 | "exception" instead of the job ID. 517 | 518 | """ 519 | client = test_client() 520 | # The 'example' job type (defined above) will crash on this invalid 521 | # time value. 522 | response = client.post( 523 | "/job/exception", 524 | data=json.dumps( 525 | {"job_type": "example", "api_key": 42, "data": {"time": "not_a_time"}} 526 | ), 527 | content_type="application/json", 528 | ) 529 | 530 | assert json.loads(response.data)["job_id"] == "exception", json.loads( 531 | response.data 532 | ) 533 | 534 | @httpretty.activate 535 | def test_get_job_with_known_error(self): 536 | """Test getting a job that failed with a JobError. 537 | 538 | Among other things, we expect the job dict to have an "error" key with 539 | the error string from the job function as its value. 540 | 541 | """ 542 | client = test_client() 543 | mock_result_url(RESULT_URL) 544 | event = _mock_test_callback_url(client) 545 | 546 | response = client.post( 547 | "/job/missing_time", 548 | data=json.dumps( 549 | { 550 | "job_type": "example", 551 | "api_key": 42, 552 | "data": {}, 553 | "result_url": RESULT_URL, 554 | } 555 | ), 556 | content_type="application/json", 557 | ) 558 | 559 | login(client) 560 | 561 | timeout = 10.0 562 | assert event.wait( 563 | timeout 564 | ), "_TEST_CALLBACK_URL was not called within {timeout} " "seconds".format( 565 | timeout=timeout 566 | ) 567 | 568 | response = client.get("/job/missing_time") 569 | 570 | job_status_data = json.loads(response.data) 571 | job_status_data.pop("requested_timestamp") 572 | job_status_data.pop("finished_timestamp") 573 | job_status_data.pop("aps_job_id") 574 | assert job_status_data == { 575 | "status": "error", 576 | "sent_data": {}, 577 | "job_id": "missing_time", 578 | "job_type": "example", 579 | "error": {"message": "time not in input"}, 580 | "data": None, 581 | "metadata": {}, 582 | "logs": [], 583 | "result_url": RESULT_URL, 584 | }, job_status_data 585 | 586 | # get_job() shouldn't return the API key, either. 587 | job_ = db.get_job(job_status_data["job_id"]) 588 | assert not job_["api_key"], job_ 589 | 590 | @httpretty.activate 591 | def test_get_job_with_unknown_error(self): 592 | """Test getting a job that failed with a random exception. 593 | 594 | A random exception type caused by an error in the job function code, 595 | as opposed to a deliberately raised JobError. 596 | 597 | """ 598 | client = test_client() 599 | mock_result_url(RESULT_URL) 600 | event = _mock_test_callback_url(client) 601 | 602 | response = client.post( 603 | "/job/exception", 604 | data=json.dumps( 605 | { 606 | "job_type": "example", 607 | "api_key": 42, 608 | "data": {"time": "not_a_time"}, 609 | "result_url": RESULT_URL, 610 | } 611 | ), 612 | content_type="application/json", 613 | ) 614 | 615 | login(client) 616 | 617 | timeout = 10.0 618 | assert event.wait( 619 | timeout 620 | ), "_TEST_CALLBACK_URL was not called within {timeout} " "seconds".format( 621 | timeout=timeout 622 | ) 623 | 624 | response = client.get("/job/exception") 625 | 626 | job_status_data = json.loads(response.data) 627 | job_status_data.pop("requested_timestamp") 628 | job_status_data.pop("finished_timestamp") 629 | job_status_data.pop("aps_job_id") 630 | 631 | error = job_status_data.pop("error") 632 | 633 | assert job_status_data == { 634 | "status": "error", 635 | "sent_data": {"time": "not_a_time"}, 636 | "job_id": "exception", 637 | "job_type": "example", 638 | "data": None, 639 | "metadata": {}, 640 | "logs": [], 641 | "result_url": RESULT_URL, 642 | }, job_status_data 643 | assert "TypeError" in error["message"], error["message"] 644 | 645 | # get_job() shouldn't return the API key, either. 646 | job_ = db.get_job(job_status_data["job_id"]) 647 | assert not job_["api_key"], job_ 648 | 649 | @httpretty.activate 650 | def test_asynchronous_post_with_result_url(self): 651 | """It should post job results to their result URLs. 652 | 653 | If a job has a result_url parameter then when the job finishes 654 | ckanserviceprovider should post the job's result to the result_url. 655 | 656 | """ 657 | client = test_client() 658 | 659 | # A thread event that we'll set when the mocked result URL is posted to 660 | event = threading.Event() 661 | 662 | # Mock the result URL. 663 | def result_url(request, uri, headers): 664 | """Handle a request to the mocked result URL.""" 665 | 666 | try: 667 | assert request.headers["content-type"] == "application/json", ( 668 | "ckanserviceprovider should post to result URLs with " 669 | "content-type application/json" 670 | ) 671 | 672 | # Check that the result URL was called with the right data. 673 | data = json.loads(request.body) 674 | data.pop("requested_timestamp") 675 | data.pop("finished_timestamp") 676 | data.pop("job_key") 677 | data.pop("aps_job_id") 678 | assert data == { 679 | "status": "complete", 680 | "sent_data": {"time": 0.1}, 681 | "job_id": "with_result", 682 | "job_type": "example", 683 | "result_url": RESULT_URL, 684 | "error": None, 685 | "data": "Slept for 0.1 seconds.", 686 | "metadata": {"key": "value"}, 687 | "logs": [], 688 | } 689 | finally: 690 | event.set() 691 | return (200, headers, request.body) 692 | 693 | httpretty.register_uri(httpretty.POST, RESULT_URL, body=result_url) 694 | 695 | response = client.post( 696 | "/job/with_result", 697 | data=json.dumps( 698 | { 699 | "job_type": "example", 700 | "data": {"time": 0.1}, 701 | "metadata": {"key": "value"}, 702 | "result_url": RESULT_URL, 703 | "api_key": "header:key", 704 | } 705 | ), 706 | content_type="application/json", 707 | ) 708 | 709 | # Wait until ckanserviceprovider has posted the result of its 710 | # asynchronous background job to the mocked result URL. 711 | timeout = 10.0 712 | assert event.wait( 713 | timeout 714 | ), "result_url was not posted to within {timeout} seconds".format( 715 | timeout=timeout 716 | ) 717 | 718 | login(client) 719 | 720 | response = client.get("/job/with_result") 721 | job_status_data = json.loads(response.data) 722 | job_status_data.pop("requested_timestamp") 723 | job_status_data.pop("finished_timestamp") 724 | job_status_data.pop("aps_job_id") 725 | 726 | assert job_status_data == { 727 | "status": "complete", 728 | "sent_data": {"time": 0.1}, 729 | "job_id": "with_result", 730 | "job_type": "example", 731 | "error": None, 732 | "data": "Slept for 0.1 seconds.", 733 | "metadata": {"key": "value"}, 734 | "logs": [], 735 | "result_url": RESULT_URL, 736 | }, job_status_data 737 | 738 | @httpretty.activate 739 | def test_asynchronous_post_with_bad_result_url(self): 740 | """It should store an error if given a bad result URL. 741 | 742 | If given an asynchronous job request with a bad result URL 743 | ckanserviceprovider should store a 744 | "Process completed but unable to post to result_url" error. 745 | 746 | This error overwrites any error that might have happened with the job 747 | itself! 748 | 749 | """ 750 | client = test_client() 751 | event = _mock_test_callback_url(client) 752 | 753 | httpretty.register_uri(httpretty.POST, RESULT_URL, status=404) 754 | 755 | response = client.post( 756 | "/job/with_bad_result", 757 | data=json.dumps( 758 | { 759 | "job_type": "example", 760 | "api_key": 42, 761 | "data": {"time": 0.1}, 762 | "metadata": {"key": "value"}, 763 | "result_url": RESULT_URL, 764 | } 765 | ), 766 | content_type="application/json", 767 | ) 768 | 769 | timeout = 10.0 770 | assert event.wait( 771 | timeout 772 | ), "_TEST_CALLBACK_URL was not called within {timeout} " "seconds".format( 773 | timeout=timeout 774 | ) 775 | 776 | login(client) 777 | response = client.get("/job/with_bad_result") 778 | job_status_data = json.loads(response.data) 779 | job_status_data.pop("requested_timestamp") 780 | job_status_data.pop("finished_timestamp") 781 | job_status_data.pop("aps_job_id") 782 | 783 | assert job_status_data == { 784 | "status": "complete", 785 | "sent_data": {"time": 0.1}, 786 | "job_id": "with_bad_result", 787 | "job_type": "example", 788 | "error": { 789 | "message": "Process completed but unable to post to " "result_url" 790 | }, 791 | "data": "Slept for 0.1 seconds.", 792 | "metadata": {"key": "value"}, 793 | "logs": [], 794 | "result_url": RESULT_URL, 795 | }, job_status_data 796 | 797 | job_ = db.get_job(job_status_data["job_id"]) 798 | assert not job_["api_key"], job_ 799 | 800 | def test_missing_job_id(self): 801 | """Trying to get a job ID that doesn't exist should return an HTTP 404. 802 | 803 | The response body should be a JSON object containing a not found error. 804 | 805 | """ 806 | client = test_client() 807 | response = client.get("/job/not_there") 808 | assert response.status_code == 404, response.status 809 | error = json.loads(response.data) 810 | assert error == {"error": "job_id not found"} 811 | 812 | def test_not_authorized_to_view_job(self): 813 | """Getting a job that you're not authorized to view should 403.""" 814 | client = test_client() 815 | response = client.post( 816 | "/job/one_job", 817 | data=json.dumps({"job_type": "echo", "api_key": 42}), 818 | content_type="application/json", 819 | ) 820 | 821 | assert response.status_code == 200, response.status 822 | job_status_data = json.loads(response.data) 823 | job_key = job_status_data["job_key"] 824 | response = client.get("/job/one_job") 825 | 826 | assert response.status_code == 403, response.status 827 | error = json.loads(response.data) 828 | assert error == {"error": "not authorized"} 829 | 830 | headers = {"Authorization": job_key} 831 | response = client.get("/job/one_job", headers=headers) 832 | assert response.status_code == 200, response.status 833 | 834 | def test_bad_metadata(self): 835 | """Posting a job with non-JSON metadata should error.""" 836 | client = test_client() 837 | response = client.post( 838 | "/job/with_bad_metadata", 839 | data=json.dumps( 840 | { 841 | "job_type": "example", 842 | "api_key": 42, 843 | "data": {"time": 0.1}, 844 | "metadata": "meta", 845 | } 846 | ), 847 | content_type="application/json", 848 | ) 849 | 850 | return_value = json.loads(response.data) 851 | assert return_value == { 852 | "error": "metadata has to be a " "json object" 853 | }, return_value 854 | 855 | def test_bad_url(self): 856 | """Posting a job with an invalid result_url should error.""" 857 | client = test_client() 858 | response = client.post( 859 | "/job/with_bad_result", 860 | data=json.dumps( 861 | { 862 | "job_type": "example", 863 | "api_key": 42, 864 | "data": {"time": 0.1}, 865 | "metadata": "meta", 866 | "result_url": "ht//0.0.0.0:9091/resul", 867 | } 868 | ), 869 | content_type="application/json", 870 | ) 871 | 872 | return_value = json.loads(response.data) 873 | assert return_value == { 874 | "error": "result_url has to start " "with http" 875 | }, return_value 876 | 877 | @pytest.mark.skip(reason="Can't change misfire_grace_time in APScheduler>3") 878 | @httpretty.activate 879 | def test_misfire(self): 880 | """Jobs should error if not completed within the misfire_grace_time.""" 881 | client = test_client() 882 | event = mock_result_url(RESULT_URL) 883 | 884 | web.scheduler.misfire_grace_time = 0.000001 885 | response = client.post( 886 | "/job/misfire", 887 | data=json.dumps( 888 | { 889 | "job_type": "example", 890 | "api_key": 42, 891 | "data": {"time": 0.1}, 892 | "metadata": { 893 | "moon": "moon", 894 | "nested": {"nested": "nested"}, 895 | "key": "value", 896 | }, 897 | "result_url": RESULT_URL, 898 | } 899 | ), 900 | content_type="application/json", 901 | ) 902 | 903 | timeout = 10.0 904 | assert event.wait( 905 | timeout 906 | ), "result_url was not called within {timeout} seconds".format(timeout=timeout) 907 | 908 | login(client) 909 | response = client.get("/job/misfire") 910 | job_status_data = json.loads(response.data) 911 | job_status_data.pop("requested_timestamp") 912 | job_status_data.pop("finished_timestamp") 913 | job_status_data.pop("aps_job_id") 914 | 915 | assert job_status_data == { 916 | "status": "error", 917 | "sent_data": {"time": 0.1}, 918 | "job_id": "misfire", 919 | "job_type": "example", 920 | "error": {"message": "Job delayed too long, service full"}, 921 | "data": None, 922 | "logs": [], 923 | "metadata": { 924 | "key": "value", 925 | "moon": "moon", 926 | "nested": {"nested": "nested"}, 927 | }, 928 | "result_url": RESULT_URL, 929 | } 930 | 931 | def test_synchronous_raw_post(self): 932 | """Posting a raw synchronous job should get result in response body. 933 | 934 | User posts a "raw" synchronous job request, ckan-service-provider runs 935 | the job and returns an HTTP response with the job result as body. 936 | (A "raw" job is one whose result is a raw text value rather than JSON 937 | text.) 938 | 939 | """ 940 | client = test_client() 941 | response = client.post( 942 | "/job/echoraw", 943 | data=json.dumps( 944 | { 945 | "metadata": {"key": "value", "moo": "moo"}, 946 | "job_type": "echo_raw", 947 | "api_key": 42, 948 | "data": "ping", 949 | } 950 | ), 951 | content_type="application/json", 952 | ) 953 | 954 | if sys.version_info[0] < 3: 955 | assert response.data == "ginp" 956 | else: 957 | assert response.data == b"ginp" 958 | 959 | def test_synchronous_post(self): 960 | """Posting a synchronous job should get a JSON response with result. 961 | 962 | User posts a synchronous job request, ckan-service-provider runs the 963 | job and returns an HTTP response with a JSON body containing the job 964 | result. 965 | 966 | """ 967 | client = test_client() 968 | response = client.post( 969 | "/job/echobasic", 970 | data=json.dumps( 971 | { 972 | "metadata": {"key": "value", "moo": "moo", "mimetype": "text/csv"}, 973 | "job_type": "echo", 974 | "api_key": 42, 975 | "data": "ping", 976 | } 977 | ), 978 | content_type="application/json", 979 | ) 980 | 981 | return_data = json.loads(response.data) 982 | return_data.pop("requested_timestamp") 983 | return_data.pop("finished_timestamp") 984 | return_data.pop("aps_job_id") 985 | job_key = return_data.pop("job_key") 986 | 987 | job_ = db.get_job(return_data["job_id"]) 988 | assert not job_["api_key"], job_ 989 | 990 | assert return_data == { 991 | "status": "complete", 992 | "sent_data": "ping", 993 | "job_id": "echobasic", 994 | "job_type": "echo", 995 | "result_url": None, 996 | "error": None, 997 | "data": ">ping", 998 | "logs": [], 999 | "metadata": {"key": "value", "moo": "moo", "mimetype": "text/csv"}, 1000 | } 1001 | 1002 | login(client) 1003 | response = client.get("/job/echobasic") 1004 | assert response.status_code == 200, response.status 1005 | job_status_data = json.loads(response.data) 1006 | job_status_data.pop("requested_timestamp") 1007 | job_status_data.pop("finished_timestamp") 1008 | job_status_data.pop("aps_job_id") 1009 | 1010 | assert return_data == job_status_data 1011 | 1012 | headers = {"Authorization": job_key} 1013 | response = client.get("/job/echobasic/data", headers=headers) 1014 | assert response.status_code == 200, response.status 1015 | 1016 | if sys.version_info[0] < 3: 1017 | assert response.data == ">ping" 1018 | else: 1019 | assert response.data == b">ping" 1020 | assert "text/csv" in response.content_type, response.content_type 1021 | 1022 | response = client.post( 1023 | "/job/echobasic", 1024 | data=json.dumps({"job_type": "echo", "api_key": 42, "data": "ping"}), 1025 | content_type="application/json", 1026 | ) 1027 | 1028 | return_data = json.loads(response.data) 1029 | assert return_data == {"error": "job_id echobasic already exists"} 1030 | 1031 | response = client.post( 1032 | "/job/echoknownbad", 1033 | data=json.dumps({"job_type": "echo", "api_key": 42, "data": ">ping"}), 1034 | content_type="application/json", 1035 | ) 1036 | assert response.status_code == 200, response.status 1037 | return_data = json.loads(response.data) 1038 | return_data.pop("requested_timestamp") 1039 | return_data.pop("finished_timestamp") 1040 | return_data.pop("job_key") 1041 | return_data.pop("aps_job_id") 1042 | assert return_data == { 1043 | "status": "error", 1044 | "sent_data": ">ping", 1045 | "job_id": "echoknownbad", 1046 | "job_type": "echo", 1047 | "result_url": None, 1048 | "error": {"message": "Do not start message with >"}, 1049 | "data": None, 1050 | "logs": [], 1051 | "metadata": {}, 1052 | } 1053 | 1054 | response = client.post( 1055 | "/job/echounknownbad", 1056 | data=json.dumps({"job_type": "echo", "api_key": 42, "data": 1}), 1057 | content_type="application/json", 1058 | ) 1059 | return_data = json.loads(response.data) 1060 | assert "AttributeError" in return_data["error"]["message"] 1061 | 1062 | response = client.post( 1063 | "/job/echobad_url", 1064 | data=json.dumps( 1065 | { 1066 | "job_type": "echo", 1067 | "api_key": 42, 1068 | "data": "moo", 1069 | "result_url": "http://bad_url", 1070 | } 1071 | ), 1072 | content_type="application/json", 1073 | ) 1074 | return_data = json.loads(response.data) 1075 | return_data.pop("requested_timestamp") 1076 | return_data.pop("finished_timestamp") 1077 | return_data.pop("job_key") 1078 | return_data.pop("aps_job_id") 1079 | assert return_data == { 1080 | "status": "complete", 1081 | "sent_data": "moo", 1082 | "job_id": "echobad_url", 1083 | "job_type": "echo", 1084 | "result_url": "http://bad_url", 1085 | "error": { 1086 | "message": "Process completed but unable to post to " "result_url" 1087 | }, 1088 | "data": ">moo", 1089 | "logs": [], 1090 | "metadata": {}, 1091 | } 1092 | 1093 | @httpretty.activate 1094 | def test_logging(self): 1095 | """Getting /job/log should return logs from the job as JSON. 1096 | 1097 | Jobs can log messages using a standard logger with a StoringHandler 1098 | attached, and users can retrieve the logged messages using the 1099 | /job/log API. 1100 | 1101 | """ 1102 | client = test_client() 1103 | event = mock_result_url(RESULT_URL) 1104 | response = client.post( 1105 | "/job/log", 1106 | data=json.dumps( 1107 | { 1108 | "metadata": {}, 1109 | "job_type": "log", 1110 | "api_key": 42, 1111 | "data": "&ping", 1112 | "result_url": RESULT_URL, 1113 | } 1114 | ), 1115 | content_type="application/json", 1116 | ) 1117 | 1118 | timeout = 10.0 1119 | assert event.wait( 1120 | timeout 1121 | ), "result_url was not called within {timeout} seconds".format(timeout=timeout) 1122 | 1123 | login(client, username="testadmin", password="wrong") 1124 | response = client.get("/job/log") 1125 | assert response.status_code == 403, response.status 1126 | 1127 | login(client) 1128 | response = client.get("/job/log") 1129 | assert response.status_code == 200, response.status 1130 | 1131 | return_data = json.loads(response.data) 1132 | logs = return_data["logs"] 1133 | assert len(logs) == 1, logs 1134 | log_ = logs[0] 1135 | log_.pop("timestamp") 1136 | log_.pop("lineno") 1137 | assert log_ == { 1138 | "level": "WARNING", 1139 | "module": "test_web", 1140 | "funcName": "log", 1141 | "message": "Just a warning", 1142 | } 1143 | 1144 | def test_delete_job(self): 1145 | """Trying to get the status of a deleted job should return 404. 1146 | 1147 | This also tests that trying to delete a job when you're not authorized 1148 | returns 403. 1149 | 1150 | """ 1151 | client = test_client() 1152 | response = client.post( 1153 | "/job/to_be_deleted", 1154 | data=json.dumps( 1155 | { 1156 | "metadata": {"foo": "bar"}, 1157 | "job_type": "echo", 1158 | "api_key": 42, 1159 | "data": "&ping", 1160 | } 1161 | ), 1162 | content_type="application/json", 1163 | ) 1164 | assert response.status_code == 200, response.status 1165 | 1166 | response = client.delete("/job/to_be_deleted") 1167 | assert response.status_code == 403, response.status 1168 | 1169 | login(client) 1170 | response = client.delete("/job/to_be_deleted") 1171 | assert response.status_code == 200, response.status 1172 | 1173 | response = client.delete("/job/to_be_deleted") 1174 | assert response.status_code == 404, response.status 1175 | 1176 | def test_getting_job_data_for_missing_job(self): 1177 | """Getting the job data for a job that doesn't exist should 404.""" 1178 | client = test_client() 1179 | login(client) 1180 | response = client.get("/job/somefoo/data") 1181 | assert response.status_code == 404, response.status 1182 | 1183 | def test_list(self): 1184 | """Tests for /job which should return a list of all the jobs. 1185 | 1186 | Tests the results from getting /job with various different limits and 1187 | filters. 1188 | 1189 | """ 1190 | client = test_client() 1191 | 1192 | db.add_pending_job( 1193 | "job_01", 1194 | str(uuid.uuid4()), 1195 | "job_type", 1196 | "result_url", 1197 | "api_key", 1198 | metadata={"key": "value"}, 1199 | ) 1200 | db.mark_job_as_completed("job_01") 1201 | db.add_pending_job( 1202 | "job_02", 1203 | str(uuid.uuid4()), 1204 | "job_type", 1205 | "result_url", 1206 | "api_key", 1207 | metadata={"key": "value", "moo": "moo"}, 1208 | ) 1209 | db.mark_job_as_completed("job_02") 1210 | db.add_pending_job( 1211 | "job_03", 1212 | str(uuid.uuid4()), 1213 | "job_type", 1214 | "result_url", 1215 | "api_key", 1216 | metadata={"key": "value", "moo": "moo"}, 1217 | ) 1218 | db.mark_job_as_completed("job_03") 1219 | db.add_pending_job( 1220 | "job_04", 1221 | str(uuid.uuid4()), 1222 | "job_type", 1223 | "result_url", 1224 | "api_key", 1225 | metadata={"key": "value"}, 1226 | ) 1227 | db.mark_job_as_completed("job_04") 1228 | db.add_pending_job( 1229 | "job_05", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1230 | ) 1231 | db.mark_job_as_completed("job_05") 1232 | db.add_pending_job( 1233 | "job_06", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1234 | ) 1235 | db.mark_job_as_completed("job_06") 1236 | db.add_pending_job( 1237 | "job_07", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1238 | ) 1239 | db.mark_job_as_completed("job_07") 1240 | db.add_pending_job( 1241 | "job_08", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1242 | ) 1243 | db.mark_job_as_completed("job_08") 1244 | db.add_pending_job( 1245 | "job_09", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1246 | ) 1247 | db.add_pending_job( 1248 | "job_10", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1249 | ) 1250 | db.add_pending_job( 1251 | "job_11", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1252 | ) 1253 | db.add_pending_job( 1254 | "job_12", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1255 | ) 1256 | db.add_pending_job( 1257 | "job_13", str(uuid.uuid4()), "job_type", "result_url", "api_key" 1258 | ) 1259 | 1260 | response = client.get("/job") 1261 | return_data = json.loads(response.data) 1262 | assert len(return_data["list"]) == 13, return_data["list"] 1263 | 1264 | response = client.get("/job?_limit=1") 1265 | return_data = json.loads(response.data) 1266 | assert len(return_data["list"]) == 1, return_data["list"] 1267 | 1268 | response = client.get("/job?_status=complete") 1269 | return_data = json.loads(response.data) 1270 | assert len(return_data["list"]) == 8, return_data["list"] 1271 | 1272 | response = client.get("/job?key=value") 1273 | return_data = json.loads(response.data) 1274 | assert len(return_data["list"]) == 4, return_data["list"] 1275 | 1276 | response = client.get("/job?key=value&moo=moo") 1277 | return_data = json.loads(response.data) 1278 | assert len(return_data["list"]) == 2, return_data["list"] 1279 | 1280 | response = client.get("/job?key=value&moo=moo&moon=moon") 1281 | return_data = json.loads(response.data) 1282 | assert len(return_data["list"]) == 0, return_data["list"] 1283 | 1284 | response = client.get("/job?key=value&moon=moon") 1285 | return_data = json.loads(response.data) 1286 | assert len(return_data["list"]) == 0, return_data["list"] 1287 | 1288 | def test_clear_all(self): 1289 | """Making a DELETE request to /job should delete all jobs. 1290 | 1291 | This also tests the 403 response when you're not authorized to delete, 1292 | and tests the ?days argument. 1293 | 1294 | """ 1295 | client = test_client() 1296 | 1297 | # Add some jobs, all completed and therefore eligible for deletion. 1298 | db.add_pending_job( 1299 | "job_01", 1300 | str(uuid.uuid4()), 1301 | "job_type", 1302 | "result_url", 1303 | "api_key", 1304 | metadata={"key": "value"}, 1305 | ) 1306 | db.mark_job_as_completed("job_01") 1307 | db.add_pending_job( 1308 | "job_02", 1309 | str(uuid.uuid4()), 1310 | "job_type", 1311 | "result_url", 1312 | "api_key", 1313 | metadata={"key": "value", "moo": "moo"}, 1314 | ) 1315 | db.mark_job_as_completed("job_02") 1316 | db.add_pending_job( 1317 | "job_03", 1318 | str(uuid.uuid4()), 1319 | "job_type", 1320 | "result_url", 1321 | "api_key", 1322 | metadata={"key": "value", "moo": "moo"}, 1323 | ) 1324 | db.mark_job_as_completed("job_03") 1325 | 1326 | original_number_of_jobs = number_of_jobs(client) 1327 | 1328 | # This should not delete any jobs because not authorized. 1329 | response = client.delete("/job") 1330 | assert response.status_code == 403, response.status 1331 | assert number_of_jobs(client) == original_number_of_jobs 1332 | 1333 | login(client) 1334 | 1335 | # This should not delete any jobs because the jobs aren't old enough. 1336 | response = client.delete("/job") 1337 | assert response.status_code == 200, response.status 1338 | assert number_of_jobs(client) == original_number_of_jobs 1339 | 1340 | # This should delete all the jobs. 1341 | response = client.delete("/job?days=0") 1342 | assert response.status_code == 200, response.status 1343 | assert number_of_jobs(client) == 0 1344 | --------------------------------------------------------------------------------