├── .github └── workflows │ ├── add-depr-ticket-to-depr-board.yml │ ├── add-remove-label-on-comment.yml │ ├── ci.yml │ ├── commitlint.yml │ ├── self-assign-issue.yml │ └── upgrade-python-requirements.yml ├── .gitignore ├── AUTHORS ├── Dockerfile ├── LICENSE.TXT ├── Makefile ├── README.md ├── conf.d ├── 600.json └── logging.json ├── grader_support ├── __init__.py ├── gradelib.py ├── graderutil.py └── run.py ├── load_test ├── mock_xqueue.py └── run.py ├── openedx.yaml ├── requirements ├── base.in ├── base.txt ├── ci.in ├── ci.txt ├── common_constraints.txt ├── constraints.txt ├── pip.in ├── pip.txt ├── pip_tools.in ├── pip_tools.txt ├── production.in ├── production.txt ├── test.in └── test.txt ├── setup.py ├── tests ├── __init__.py ├── fixtures │ ├── __init__.py │ ├── answer.py │ ├── config │ │ ├── conf.d │ │ │ └── example.json │ │ └── logging.json │ └── fake_grader.py ├── test_grader.py ├── test_jailed_grader.py ├── test_manager.py └── test_xqueue_client.py └── xqueue_watcher ├── __init__.py ├── __main__.py ├── client.py ├── grader.py ├── jailedgrader.py ├── manager.py └── settings.py /.github/workflows/add-depr-ticket-to-depr-board.yml: -------------------------------------------------------------------------------- 1 | # Run the workflow that adds new tickets that are either: 2 | # - labelled "DEPR" 3 | # - title starts with "[DEPR]" 4 | # - body starts with "Proposal Date" (this is the first template field) 5 | # to the org-wide DEPR project board 6 | 7 | name: Add newly created DEPR issues to the DEPR project board 8 | 9 | on: 10 | issues: 11 | types: [opened] 12 | 13 | jobs: 14 | routeissue: 15 | uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master 16 | secrets: 17 | GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }} 18 | GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }} 19 | SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }} 20 | -------------------------------------------------------------------------------- /.github/workflows/add-remove-label-on-comment.yml: -------------------------------------------------------------------------------- 1 | # This workflow runs when a comment is made on the ticket 2 | # If the comment starts with "label: " it tries to apply 3 | # the label indicated in rest of comment. 4 | # If the comment starts with "remove label: ", it tries 5 | # to remove the indicated label. 6 | # Note: Labels are allowed to have spaces and this script does 7 | # not parse spaces (as often a space is legitimate), so the command 8 | # "label: really long lots of words label" will apply the 9 | # label "really long lots of words label" 10 | 11 | name: Allows for the adding and removing of labels via comment 12 | 13 | on: 14 | issue_comment: 15 | types: [created] 16 | 17 | jobs: 18 | add_remove_labels: 19 | uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master 20 | 21 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Python CI 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - '**' 10 | 11 | jobs: 12 | run_tests: 13 | name: Tests 14 | runs-on: ${{ matrix.os }} 15 | strategy: 16 | matrix: 17 | os: 18 | - ubuntu-latest 19 | python-version: ['3.11', '3.12'] 20 | steps: 21 | - uses: actions/checkout@v4 22 | - name: setup python 23 | uses: actions/setup-python@v5 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | 27 | - name: Install requirements and Run Tests 28 | run: make test 29 | 30 | - name: Run Coverage 31 | uses: codecov/codecov-action@v4 32 | with: 33 | token: ${{ secrets.CODECOV_TOKEN }} 34 | fail_ci_if_error: true 35 | -------------------------------------------------------------------------------- /.github/workflows/commitlint.yml: -------------------------------------------------------------------------------- 1 | # Run commitlint on the commit messages in a pull request. 2 | 3 | name: Lint Commit Messages 4 | 5 | on: 6 | - pull_request 7 | 8 | jobs: 9 | commitlint: 10 | uses: openedx/.github/.github/workflows/commitlint.yml@master 11 | -------------------------------------------------------------------------------- /.github/workflows/self-assign-issue.yml: -------------------------------------------------------------------------------- 1 | # This workflow runs when a comment is made on the ticket 2 | # If the comment starts with "assign me" it assigns the author to the 3 | # ticket (case insensitive) 4 | 5 | name: Assign comment author to ticket if they say "assign me" 6 | on: 7 | issue_comment: 8 | types: [created] 9 | 10 | jobs: 11 | self_assign_by_comment: 12 | uses: openedx/.github/.github/workflows/self-assign-issue.yml@master 13 | -------------------------------------------------------------------------------- /.github/workflows/upgrade-python-requirements.yml: -------------------------------------------------------------------------------- 1 | name: Upgrade Python Requirements 2 | 3 | on: 4 | schedule: 5 | - cron: "15 15 1/14 * *" 6 | workflow_dispatch: 7 | inputs: 8 | branch: 9 | description: "Target branch against which to create requirements PR" 10 | required: true 11 | default: 'master' 12 | 13 | jobs: 14 | call-upgrade-python-requirements-workflow: 15 | uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master 16 | with: 17 | branch: ${{ github.event.inputs.branch || 'master' }} 18 | # optional parameters below; fill in if you'd like github or email notifications 19 | # user_reviewers: "" 20 | # team_reviewers: "" 21 | email_address: "aurora-requirements-update@2u-internal.opsgenie.net" 22 | send_success_notification: true 23 | secrets: 24 | requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} 25 | requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} 26 | edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} 27 | edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | *.scssc 4 | *.swp 5 | *.orig 6 | *.DS_Store 7 | :2e_* 8 | :2e# 9 | .AppleDouble 10 | database.sqlite 11 | courseware/static/js/mathjax/* 12 | db.newaskbot 13 | db.oldaskbot 14 | flushdb.sh 15 | build 16 | .coverage 17 | coverage.xml 18 | cover/ 19 | log/ 20 | reports/ 21 | /src/ 22 | \#*\# 23 | *.egg-info 24 | .idea/ 25 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Isaac Chuang 2 | Tomas Lozano-Perez 3 | Adam Hartz 4 | Tony Kim 5 | Lyla Fischer 6 | Calen Pennington 7 | Victor Shnayder 8 | David Ormsbee 9 | Carlos Andrés Rocha 10 | Ned Batchelder 11 | John Jarvis 12 | Ashley Penney 13 | Sarina Canelake 14 | Will Daly 15 | James Tauber 16 | Dave St.Germain 17 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial as openedx 2 | 3 | RUN apt update && \ 4 | apt install -y git-core language-pack-en apparmor apparmor-utils python python-pip python-dev && \ 5 | pip install --upgrade pip setuptools && \ 6 | rm -rf /var/lib/apt/lists/* 7 | 8 | RUN locale-gen en_US.UTF-8 9 | ENV LANG en_US.UTF-8 10 | ENV LANGUAGE en_US:en 11 | ENV LC_ALL en_US.UTF-8 12 | 13 | WORKDIR /edx/app/xqueue_watcher 14 | COPY requirements /edx/app/xqueue_watcher/requirements 15 | RUN pip install -r requirements/production.txt 16 | 17 | CMD python -m xqueue_watcher -d /edx/etc/xqueue_watcher 18 | 19 | RUN useradd -m --shell /bin/false app 20 | USER app 21 | 22 | COPY . /edx/app/xqueue_watcher 23 | 24 | FROM openedx as edx.org 25 | RUN pip install newrelic 26 | CMD newrelic-admin run-program python -m xqueue_watcher -d /edx/etc/xqueue_watcher 27 | -------------------------------------------------------------------------------- /LICENSE.TXT: -------------------------------------------------------------------------------- 1 | GNU AFFERO GENERAL PUBLIC LICENSE 2 | Version 3, 19 November 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | The GNU Affero General Public License is a free, copyleft license for 11 | software and other kinds of works, specifically designed to ensure 12 | cooperation with the community in the case of network server software. 13 | 14 | The licenses for most software and other practical works are designed 15 | to take away your freedom to share and change the works. By contrast, 16 | our General Public Licenses are intended to guarantee your freedom to 17 | share and change all versions of a program--to make sure it remains free 18 | software for all its users. 19 | 20 | When we speak of free software, we are referring to freedom, not 21 | price. Our General Public Licenses are designed to make sure that you 22 | have the freedom to distribute copies of free software (and charge for 23 | them if you wish), that you receive source code or can get it if you 24 | want it, that you can change the software or use pieces of it in new 25 | free programs, and that you know you can do these things. 26 | 27 | Developers that use our General Public Licenses protect your rights 28 | with two steps: (1) assert copyright on the software, and (2) offer 29 | you this License which gives you legal permission to copy, distribute 30 | and/or modify the software. 31 | 32 | A secondary benefit of defending all users' freedom is that 33 | improvements made in alternate versions of the program, if they 34 | receive widespread use, become available for other developers to 35 | incorporate. Many developers of free software are heartened and 36 | encouraged by the resulting cooperation. However, in the case of 37 | software used on network servers, this result may fail to come about. 38 | The GNU General Public License permits making a modified version and 39 | letting the public access it on a server without ever releasing its 40 | source code to the public. 41 | 42 | The GNU Affero General Public License is designed specifically to 43 | ensure that, in such cases, the modified source code becomes available 44 | to the community. It requires the operator of a network server to 45 | provide the source code of the modified version running there to the 46 | users of that server. Therefore, public use of a modified version, on 47 | a publicly accessible server, gives the public access to the source 48 | code of the modified version. 49 | 50 | An older license, called the Affero General Public License and 51 | published by Affero, was designed to accomplish similar goals. This is 52 | a different license, not a version of the Affero GPL, but Affero has 53 | released a new version of the Affero GPL which permits relicensing under 54 | this license. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | TERMS AND CONDITIONS 60 | 61 | 0. Definitions. 62 | 63 | "This License" refers to version 3 of the GNU Affero General Public License. 64 | 65 | "Copyright" also means copyright-like laws that apply to other kinds of 66 | works, such as semiconductor masks. 67 | 68 | "The Program" refers to any copyrightable work licensed under this 69 | License. Each licensee is addressed as "you". "Licensees" and 70 | "recipients" may be individuals or organizations. 71 | 72 | To "modify" a work means to copy from or adapt all or part of the work 73 | in a fashion requiring copyright permission, other than the making of an 74 | exact copy. The resulting work is called a "modified version" of the 75 | earlier work or a work "based on" the earlier work. 76 | 77 | A "covered work" means either the unmodified Program or a work based 78 | on the Program. 79 | 80 | To "propagate" a work means to do anything with it that, without 81 | permission, would make you directly or secondarily liable for 82 | infringement under applicable copyright law, except executing it on a 83 | computer or modifying a private copy. Propagation includes copying, 84 | distribution (with or without modification), making available to the 85 | public, and in some countries other activities as well. 86 | 87 | To "convey" a work means any kind of propagation that enables other 88 | parties to make or receive copies. Mere interaction with a user through 89 | a computer network, with no transfer of a copy, is not conveying. 90 | 91 | An interactive user interface displays "Appropriate Legal Notices" 92 | to the extent that it includes a convenient and prominently visible 93 | feature that (1) displays an appropriate copyright notice, and (2) 94 | tells the user that there is no warranty for the work (except to the 95 | extent that warranties are provided), that licensees may convey the 96 | work under this License, and how to view a copy of this License. If 97 | the interface presents a list of user commands or options, such as a 98 | menu, a prominent item in the list meets this criterion. 99 | 100 | 1. Source Code. 101 | 102 | The "source code" for a work means the preferred form of the work 103 | for making modifications to it. "Object code" means any non-source 104 | form of a work. 105 | 106 | A "Standard Interface" means an interface that either is an official 107 | standard defined by a recognized standards body, or, in the case of 108 | interfaces specified for a particular programming language, one that 109 | is widely used among developers working in that language. 110 | 111 | The "System Libraries" of an executable work include anything, other 112 | than the work as a whole, that (a) is included in the normal form of 113 | packaging a Major Component, but which is not part of that Major 114 | Component, and (b) serves only to enable use of the work with that 115 | Major Component, or to implement a Standard Interface for which an 116 | implementation is available to the public in source code form. A 117 | "Major Component", in this context, means a major essential component 118 | (kernel, window system, and so on) of the specific operating system 119 | (if any) on which the executable work runs, or a compiler used to 120 | produce the work, or an object code interpreter used to run it. 121 | 122 | The "Corresponding Source" for a work in object code form means all 123 | the source code needed to generate, install, and (for an executable 124 | work) run the object code and to modify the work, including scripts to 125 | control those activities. However, it does not include the work's 126 | System Libraries, or general-purpose tools or generally available free 127 | programs which are used unmodified in performing those activities but 128 | which are not part of the work. For example, Corresponding Source 129 | includes interface definition files associated with source files for 130 | the work, and the source code for shared libraries and dynamically 131 | linked subprograms that the work is specifically designed to require, 132 | such as by intimate data communication or control flow between those 133 | subprograms and other parts of the work. 134 | 135 | The Corresponding Source need not include anything that users 136 | can regenerate automatically from other parts of the Corresponding 137 | Source. 138 | 139 | The Corresponding Source for a work in source code form is that 140 | same work. 141 | 142 | 2. Basic Permissions. 143 | 144 | All rights granted under this License are granted for the term of 145 | copyright on the Program, and are irrevocable provided the stated 146 | conditions are met. This License explicitly affirms your unlimited 147 | permission to run the unmodified Program. The output from running a 148 | covered work is covered by this License only if the output, given its 149 | content, constitutes a covered work. This License acknowledges your 150 | rights of fair use or other equivalent, as provided by copyright law. 151 | 152 | You may make, run and propagate covered works that you do not 153 | convey, without conditions so long as your license otherwise remains 154 | in force. You may convey covered works to others for the sole purpose 155 | of having them make modifications exclusively for you, or provide you 156 | with facilities for running those works, provided that you comply with 157 | the terms of this License in conveying all material for which you do 158 | not control copyright. Those thus making or running the covered works 159 | for you must do so exclusively on your behalf, under your direction 160 | and control, on terms that prohibit them from making any copies of 161 | your copyrighted material outside their relationship with you. 162 | 163 | Conveying under any other circumstances is permitted solely under 164 | the conditions stated below. Sublicensing is not allowed; section 10 165 | makes it unnecessary. 166 | 167 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 168 | 169 | No covered work shall be deemed part of an effective technological 170 | measure under any applicable law fulfilling obligations under article 171 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 172 | similar laws prohibiting or restricting circumvention of such 173 | measures. 174 | 175 | When you convey a covered work, you waive any legal power to forbid 176 | circumvention of technological measures to the extent such circumvention 177 | is effected by exercising rights under this License with respect to 178 | the covered work, and you disclaim any intention to limit operation or 179 | modification of the work as a means of enforcing, against the work's 180 | users, your or third parties' legal rights to forbid circumvention of 181 | technological measures. 182 | 183 | 4. Conveying Verbatim Copies. 184 | 185 | You may convey verbatim copies of the Program's source code as you 186 | receive it, in any medium, provided that you conspicuously and 187 | appropriately publish on each copy an appropriate copyright notice; 188 | keep intact all notices stating that this License and any 189 | non-permissive terms added in accord with section 7 apply to the code; 190 | keep intact all notices of the absence of any warranty; and give all 191 | recipients a copy of this License along with the Program. 192 | 193 | You may charge any price or no price for each copy that you convey, 194 | and you may offer support or warranty protection for a fee. 195 | 196 | 5. Conveying Modified Source Versions. 197 | 198 | You may convey a work based on the Program, or the modifications to 199 | produce it from the Program, in the form of source code under the 200 | terms of section 4, provided that you also meet all of these conditions: 201 | 202 | a) The work must carry prominent notices stating that you modified 203 | it, and giving a relevant date. 204 | 205 | b) The work must carry prominent notices stating that it is 206 | released under this License and any conditions added under section 207 | 7. This requirement modifies the requirement in section 4 to 208 | "keep intact all notices". 209 | 210 | c) You must license the entire work, as a whole, under this 211 | License to anyone who comes into possession of a copy. This 212 | License will therefore apply, along with any applicable section 7 213 | additional terms, to the whole of the work, and all its parts, 214 | regardless of how they are packaged. This License gives no 215 | permission to license the work in any other way, but it does not 216 | invalidate such permission if you have separately received it. 217 | 218 | d) If the work has interactive user interfaces, each must display 219 | Appropriate Legal Notices; however, if the Program has interactive 220 | interfaces that do not display Appropriate Legal Notices, your 221 | work need not make them do so. 222 | 223 | A compilation of a covered work with other separate and independent 224 | works, which are not by their nature extensions of the covered work, 225 | and which are not combined with it such as to form a larger program, 226 | in or on a volume of a storage or distribution medium, is called an 227 | "aggregate" if the compilation and its resulting copyright are not 228 | used to limit the access or legal rights of the compilation's users 229 | beyond what the individual works permit. Inclusion of a covered work 230 | in an aggregate does not cause this License to apply to the other 231 | parts of the aggregate. 232 | 233 | 6. Conveying Non-Source Forms. 234 | 235 | You may convey a covered work in object code form under the terms 236 | of sections 4 and 5, provided that you also convey the 237 | machine-readable Corresponding Source under the terms of this License, 238 | in one of these ways: 239 | 240 | a) Convey the object code in, or embodied in, a physical product 241 | (including a physical distribution medium), accompanied by the 242 | Corresponding Source fixed on a durable physical medium 243 | customarily used for software interchange. 244 | 245 | b) Convey the object code in, or embodied in, a physical product 246 | (including a physical distribution medium), accompanied by a 247 | written offer, valid for at least three years and valid for as 248 | long as you offer spare parts or customer support for that product 249 | model, to give anyone who possesses the object code either (1) a 250 | copy of the Corresponding Source for all the software in the 251 | product that is covered by this License, on a durable physical 252 | medium customarily used for software interchange, for a price no 253 | more than your reasonable cost of physically performing this 254 | conveying of source, or (2) access to copy the 255 | Corresponding Source from a network server at no charge. 256 | 257 | c) Convey individual copies of the object code with a copy of the 258 | written offer to provide the Corresponding Source. This 259 | alternative is allowed only occasionally and noncommercially, and 260 | only if you received the object code with such an offer, in accord 261 | with subsection 6b. 262 | 263 | d) Convey the object code by offering access from a designated 264 | place (gratis or for a charge), and offer equivalent access to the 265 | Corresponding Source in the same way through the same place at no 266 | further charge. You need not require recipients to copy the 267 | Corresponding Source along with the object code. If the place to 268 | copy the object code is a network server, the Corresponding Source 269 | may be on a different server (operated by you or a third party) 270 | that supports equivalent copying facilities, provided you maintain 271 | clear directions next to the object code saying where to find the 272 | Corresponding Source. Regardless of what server hosts the 273 | Corresponding Source, you remain obligated to ensure that it is 274 | available for as long as needed to satisfy these requirements. 275 | 276 | e) Convey the object code using peer-to-peer transmission, provided 277 | you inform other peers where the object code and Corresponding 278 | Source of the work are being offered to the general public at no 279 | charge under subsection 6d. 280 | 281 | A separable portion of the object code, whose source code is excluded 282 | from the Corresponding Source as a System Library, need not be 283 | included in conveying the object code work. 284 | 285 | A "User Product" is either (1) a "consumer product", which means any 286 | tangible personal property which is normally used for personal, family, 287 | or household purposes, or (2) anything designed or sold for incorporation 288 | into a dwelling. In determining whether a product is a consumer product, 289 | doubtful cases shall be resolved in favor of coverage. For a particular 290 | product received by a particular user, "normally used" refers to a 291 | typical or common use of that class of product, regardless of the status 292 | of the particular user or of the way in which the particular user 293 | actually uses, or expects or is expected to use, the product. A product 294 | is a consumer product regardless of whether the product has substantial 295 | commercial, industrial or non-consumer uses, unless such uses represent 296 | the only significant mode of use of the product. 297 | 298 | "Installation Information" for a User Product means any methods, 299 | procedures, authorization keys, or other information required to install 300 | and execute modified versions of a covered work in that User Product from 301 | a modified version of its Corresponding Source. The information must 302 | suffice to ensure that the continued functioning of the modified object 303 | code is in no case prevented or interfered with solely because 304 | modification has been made. 305 | 306 | If you convey an object code work under this section in, or with, or 307 | specifically for use in, a User Product, and the conveying occurs as 308 | part of a transaction in which the right of possession and use of the 309 | User Product is transferred to the recipient in perpetuity or for a 310 | fixed term (regardless of how the transaction is characterized), the 311 | Corresponding Source conveyed under this section must be accompanied 312 | by the Installation Information. But this requirement does not apply 313 | if neither you nor any third party retains the ability to install 314 | modified object code on the User Product (for example, the work has 315 | been installed in ROM). 316 | 317 | The requirement to provide Installation Information does not include a 318 | requirement to continue to provide support service, warranty, or updates 319 | for a work that has been modified or installed by the recipient, or for 320 | the User Product in which it has been modified or installed. Access to a 321 | network may be denied when the modification itself materially and 322 | adversely affects the operation of the network or violates the rules and 323 | protocols for communication across the network. 324 | 325 | Corresponding Source conveyed, and Installation Information provided, 326 | in accord with this section must be in a format that is publicly 327 | documented (and with an implementation available to the public in 328 | source code form), and must require no special password or key for 329 | unpacking, reading or copying. 330 | 331 | 7. Additional Terms. 332 | 333 | "Additional permissions" are terms that supplement the terms of this 334 | License by making exceptions from one or more of its conditions. 335 | Additional permissions that are applicable to the entire Program shall 336 | be treated as though they were included in this License, to the extent 337 | that they are valid under applicable law. If additional permissions 338 | apply only to part of the Program, that part may be used separately 339 | under those permissions, but the entire Program remains governed by 340 | this License without regard to the additional permissions. 341 | 342 | When you convey a copy of a covered work, you may at your option 343 | remove any additional permissions from that copy, or from any part of 344 | it. (Additional permissions may be written to require their own 345 | removal in certain cases when you modify the work.) You may place 346 | additional permissions on material, added by you to a covered work, 347 | for which you have or can give appropriate copyright permission. 348 | 349 | Notwithstanding any other provision of this License, for material you 350 | add to a covered work, you may (if authorized by the copyright holders of 351 | that material) supplement the terms of this License with terms: 352 | 353 | a) Disclaiming warranty or limiting liability differently from the 354 | terms of sections 15 and 16 of this License; or 355 | 356 | b) Requiring preservation of specified reasonable legal notices or 357 | author attributions in that material or in the Appropriate Legal 358 | Notices displayed by works containing it; or 359 | 360 | c) Prohibiting misrepresentation of the origin of that material, or 361 | requiring that modified versions of such material be marked in 362 | reasonable ways as different from the original version; or 363 | 364 | d) Limiting the use for publicity purposes of names of licensors or 365 | authors of the material; or 366 | 367 | e) Declining to grant rights under trademark law for use of some 368 | trade names, trademarks, or service marks; or 369 | 370 | f) Requiring indemnification of licensors and authors of that 371 | material by anyone who conveys the material (or modified versions of 372 | it) with contractual assumptions of liability to the recipient, for 373 | any liability that these contractual assumptions directly impose on 374 | those licensors and authors. 375 | 376 | All other non-permissive additional terms are considered "further 377 | restrictions" within the meaning of section 10. If the Program as you 378 | received it, or any part of it, contains a notice stating that it is 379 | governed by this License along with a term that is a further 380 | restriction, you may remove that term. If a license document contains 381 | a further restriction but permits relicensing or conveying under this 382 | License, you may add to a covered work material governed by the terms 383 | of that license document, provided that the further restriction does 384 | not survive such relicensing or conveying. 385 | 386 | If you add terms to a covered work in accord with this section, you 387 | must place, in the relevant source files, a statement of the 388 | additional terms that apply to those files, or a notice indicating 389 | where to find the applicable terms. 390 | 391 | Additional terms, permissive or non-permissive, may be stated in the 392 | form of a separately written license, or stated as exceptions; 393 | the above requirements apply either way. 394 | 395 | 8. Termination. 396 | 397 | You may not propagate or modify a covered work except as expressly 398 | provided under this License. Any attempt otherwise to propagate or 399 | modify it is void, and will automatically terminate your rights under 400 | this License (including any patent licenses granted under the third 401 | paragraph of section 11). 402 | 403 | However, if you cease all violation of this License, then your 404 | license from a particular copyright holder is reinstated (a) 405 | provisionally, unless and until the copyright holder explicitly and 406 | finally terminates your license, and (b) permanently, if the copyright 407 | holder fails to notify you of the violation by some reasonable means 408 | prior to 60 days after the cessation. 409 | 410 | Moreover, your license from a particular copyright holder is 411 | reinstated permanently if the copyright holder notifies you of the 412 | violation by some reasonable means, this is the first time you have 413 | received notice of violation of this License (for any work) from that 414 | copyright holder, and you cure the violation prior to 30 days after 415 | your receipt of the notice. 416 | 417 | Termination of your rights under this section does not terminate the 418 | licenses of parties who have received copies or rights from you under 419 | this License. If your rights have been terminated and not permanently 420 | reinstated, you do not qualify to receive new licenses for the same 421 | material under section 10. 422 | 423 | 9. Acceptance Not Required for Having Copies. 424 | 425 | You are not required to accept this License in order to receive or 426 | run a copy of the Program. Ancillary propagation of a covered work 427 | occurring solely as a consequence of using peer-to-peer transmission 428 | to receive a copy likewise does not require acceptance. However, 429 | nothing other than this License grants you permission to propagate or 430 | modify any covered work. These actions infringe copyright if you do 431 | not accept this License. Therefore, by modifying or propagating a 432 | covered work, you indicate your acceptance of this License to do so. 433 | 434 | 10. Automatic Licensing of Downstream Recipients. 435 | 436 | Each time you convey a covered work, the recipient automatically 437 | receives a license from the original licensors, to run, modify and 438 | propagate that work, subject to this License. You are not responsible 439 | for enforcing compliance by third parties with this License. 440 | 441 | An "entity transaction" is a transaction transferring control of an 442 | organization, or substantially all assets of one, or subdividing an 443 | organization, or merging organizations. If propagation of a covered 444 | work results from an entity transaction, each party to that 445 | transaction who receives a copy of the work also receives whatever 446 | licenses to the work the party's predecessor in interest had or could 447 | give under the previous paragraph, plus a right to possession of the 448 | Corresponding Source of the work from the predecessor in interest, if 449 | the predecessor has it or can get it with reasonable efforts. 450 | 451 | You may not impose any further restrictions on the exercise of the 452 | rights granted or affirmed under this License. For example, you may 453 | not impose a license fee, royalty, or other charge for exercise of 454 | rights granted under this License, and you may not initiate litigation 455 | (including a cross-claim or counterclaim in a lawsuit) alleging that 456 | any patent claim is infringed by making, using, selling, offering for 457 | sale, or importing the Program or any portion of it. 458 | 459 | 11. Patents. 460 | 461 | A "contributor" is a copyright holder who authorizes use under this 462 | License of the Program or a work on which the Program is based. The 463 | work thus licensed is called the contributor's "contributor version". 464 | 465 | A contributor's "essential patent claims" are all patent claims 466 | owned or controlled by the contributor, whether already acquired or 467 | hereafter acquired, that would be infringed by some manner, permitted 468 | by this License, of making, using, or selling its contributor version, 469 | but do not include claims that would be infringed only as a 470 | consequence of further modification of the contributor version. For 471 | purposes of this definition, "control" includes the right to grant 472 | patent sublicenses in a manner consistent with the requirements of 473 | this License. 474 | 475 | Each contributor grants you a non-exclusive, worldwide, royalty-free 476 | patent license under the contributor's essential patent claims, to 477 | make, use, sell, offer for sale, import and otherwise run, modify and 478 | propagate the contents of its contributor version. 479 | 480 | In the following three paragraphs, a "patent license" is any express 481 | agreement or commitment, however denominated, not to enforce a patent 482 | (such as an express permission to practice a patent or covenant not to 483 | sue for patent infringement). To "grant" such a patent license to a 484 | party means to make such an agreement or commitment not to enforce a 485 | patent against the party. 486 | 487 | If you convey a covered work, knowingly relying on a patent license, 488 | and the Corresponding Source of the work is not available for anyone 489 | to copy, free of charge and under the terms of this License, through a 490 | publicly available network server or other readily accessible means, 491 | then you must either (1) cause the Corresponding Source to be so 492 | available, or (2) arrange to deprive yourself of the benefit of the 493 | patent license for this particular work, or (3) arrange, in a manner 494 | consistent with the requirements of this License, to extend the patent 495 | license to downstream recipients. "Knowingly relying" means you have 496 | actual knowledge that, but for the patent license, your conveying the 497 | covered work in a country, or your recipient's use of the covered work 498 | in a country, would infringe one or more identifiable patents in that 499 | country that you have reason to believe are valid. 500 | 501 | If, pursuant to or in connection with a single transaction or 502 | arrangement, you convey, or propagate by procuring conveyance of, a 503 | covered work, and grant a patent license to some of the parties 504 | receiving the covered work authorizing them to use, propagate, modify 505 | or convey a specific copy of the covered work, then the patent license 506 | you grant is automatically extended to all recipients of the covered 507 | work and works based on it. 508 | 509 | A patent license is "discriminatory" if it does not include within 510 | the scope of its coverage, prohibits the exercise of, or is 511 | conditioned on the non-exercise of one or more of the rights that are 512 | specifically granted under this License. You may not convey a covered 513 | work if you are a party to an arrangement with a third party that is 514 | in the business of distributing software, under which you make payment 515 | to the third party based on the extent of your activity of conveying 516 | the work, and under which the third party grants, to any of the 517 | parties who would receive the covered work from you, a discriminatory 518 | patent license (a) in connection with copies of the covered work 519 | conveyed by you (or copies made from those copies), or (b) primarily 520 | for and in connection with specific products or compilations that 521 | contain the covered work, unless you entered into that arrangement, 522 | or that patent license was granted, prior to 28 March 2007. 523 | 524 | Nothing in this License shall be construed as excluding or limiting 525 | any implied license or other defenses to infringement that may 526 | otherwise be available to you under applicable patent law. 527 | 528 | 12. No Surrender of Others' Freedom. 529 | 530 | If conditions are imposed on you (whether by court order, agreement or 531 | otherwise) that contradict the conditions of this License, they do not 532 | excuse you from the conditions of this License. If you cannot convey a 533 | covered work so as to satisfy simultaneously your obligations under this 534 | License and any other pertinent obligations, then as a consequence you may 535 | not convey it at all. For example, if you agree to terms that obligate you 536 | to collect a royalty for further conveying from those to whom you convey 537 | the Program, the only way you could satisfy both those terms and this 538 | License would be to refrain entirely from conveying the Program. 539 | 540 | 13. Remote Network Interaction; Use with the GNU General Public License. 541 | 542 | Notwithstanding any other provision of this License, if you modify the 543 | Program, your modified version must prominently offer all users 544 | interacting with it remotely through a computer network (if your version 545 | supports such interaction) an opportunity to receive the Corresponding 546 | Source of your version by providing access to the Corresponding Source 547 | from a network server at no charge, through some standard or customary 548 | means of facilitating copying of software. This Corresponding Source 549 | shall include the Corresponding Source for any work covered by version 3 550 | of the GNU General Public License that is incorporated pursuant to the 551 | following paragraph. 552 | 553 | Notwithstanding any other provision of this License, you have 554 | permission to link or combine any covered work with a work licensed 555 | under version 3 of the GNU General Public License into a single 556 | combined work, and to convey the resulting work. The terms of this 557 | License will continue to apply to the part which is the covered work, 558 | but the work with which it is combined will remain governed by version 559 | 3 of the GNU General Public License. 560 | 561 | 14. Revised Versions of this License. 562 | 563 | The Free Software Foundation may publish revised and/or new versions of 564 | the GNU Affero General Public License from time to time. Such new versions 565 | will be similar in spirit to the present version, but may differ in detail to 566 | address new problems or concerns. 567 | 568 | Each version is given a distinguishing version number. If the 569 | Program specifies that a certain numbered version of the GNU Affero General 570 | Public License "or any later version" applies to it, you have the 571 | option of following the terms and conditions either of that numbered 572 | version or of any later version published by the Free Software 573 | Foundation. If the Program does not specify a version number of the 574 | GNU Affero General Public License, you may choose any version ever published 575 | by the Free Software Foundation. 576 | 577 | If the Program specifies that a proxy can decide which future 578 | versions of the GNU Affero General Public License can be used, that proxy's 579 | public statement of acceptance of a version permanently authorizes you 580 | to choose that version for the Program. 581 | 582 | Later license versions may give you additional or different 583 | permissions. However, no additional obligations are imposed on any 584 | author or copyright holder as a result of your choosing to follow a 585 | later version. 586 | 587 | 15. Disclaimer of Warranty. 588 | 589 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 590 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 591 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 592 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 593 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 594 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 595 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 596 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 597 | 598 | 16. Limitation of Liability. 599 | 600 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 601 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 602 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 603 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 604 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 605 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 606 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 607 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 608 | SUCH DAMAGES. 609 | 610 | 17. Interpretation of Sections 15 and 16. 611 | 612 | If the disclaimer of warranty and limitation of liability provided 613 | above cannot be given local legal effect according to their terms, 614 | reviewing courts shall apply local law that most closely approximates 615 | an absolute waiver of all civil liability in connection with the 616 | Program, unless a warranty or assumption of liability accompanies a 617 | copy of the Program in return for a fee. 618 | 619 | END OF TERMS AND CONDITIONS 620 | 621 | How to Apply These Terms to Your New Programs 622 | 623 | If you develop a new program, and you want it to be of the greatest 624 | possible use to the public, the best way to achieve this is to make it 625 | free software which everyone can redistribute and change under these terms. 626 | 627 | To do so, attach the following notices to the program. It is safest 628 | to attach them to the start of each source file to most effectively 629 | state the exclusion of warranty; and each file should have at least 630 | the "copyright" line and a pointer to where the full notice is found. 631 | 632 | 633 | Copyright (C) 634 | 635 | This program is free software: you can redistribute it and/or modify 636 | it under the terms of the GNU Affero General Public License as published by 637 | the Free Software Foundation, either version 3 of the License, or 638 | (at your option) any later version. 639 | 640 | This program is distributed in the hope that it will be useful, 641 | but WITHOUT ANY WARRANTY; without even the implied warranty of 642 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 643 | GNU Affero General Public License for more details. 644 | 645 | You should have received a copy of the GNU Affero General Public License 646 | along with this program. If not, see . 647 | 648 | Also add information on how to contact you by electronic and paper mail. 649 | 650 | If your software can interact with users remotely through a computer 651 | network, you should also make sure that it provides a way for users to 652 | get its source. For example, if your program is a web application, its 653 | interface could display a "Source" link that leads users to an archive 654 | of the code. There are many ways you could offer source, and different 655 | solutions will be better for different programs; see section 13 for the 656 | specific requirements. 657 | 658 | You should also get your employer (if you work as a programmer) or school, 659 | if any, to sign a "copyright disclaimer" for the program, if necessary. 660 | For more information on this, and how to apply and follow the GNU AGPL, see 661 | . 662 | 663 | EdX Inc. wishes to state, in clarification of the above license terms, that 664 | any public, independently available web service offered over the network and 665 | communicating with edX's copyrighted works by any form of inter-service 666 | communication, including but not limited to Remote Procedure Call (RPC) 667 | interfaces, is not a work based on our copyrighted work within the meaning 668 | of the license. "Corresponding Source" of this work, or works based on this 669 | work, as defined by the terms of this license do not include source code 670 | files for programs used solely to provide those public, independently 671 | available web services. 672 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | NODE_BIN=./node_modules/.bin 2 | 3 | help: 4 | @echo ' ' 5 | @echo 'Makefile for the xqueue-watcher ' 6 | @echo ' ' 7 | @echo 'Usage: ' 8 | @echo ' make requirements install requirements for local development ' 9 | @echo ' make test run python unit-tests ' 10 | @echo ' make clean delete generated byte code and coverage reports ' 11 | @echo ' ' 12 | 13 | COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt 14 | .PHONY: $(COMMON_CONSTRAINTS_TXT) 15 | $(COMMON_CONSTRAINTS_TXT): 16 | wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)" 17 | 18 | upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade 19 | upgrade: $(COMMON_CONSTRAINTS_TXT) 20 | ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in 21 | pip install -q -r requirements/pip_tools.txt 22 | pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in 23 | pip-compile --upgrade -o requirements/pip_tools.txt requirements/pip_tools.in 24 | pip install -q -r requirements/pip.txt 25 | pip install -q -r requirements/pip_tools.txt 26 | pip-compile --upgrade -o requirements/base.txt requirements/base.in 27 | pip-compile --upgrade -o requirements/production.txt requirements/production.in 28 | pip-compile --upgrade -o requirements/test.txt requirements/test.in 29 | pip-compile --upgrade -o requirements/ci.txt requirements/ci.in 30 | 31 | requirements: 32 | pip install -qr requirements/production.txt --exists-action w 33 | 34 | test.requirements: 35 | pip install -q -r requirements/test.txt --exists-action w 36 | 37 | ci.requirements: 38 | pip install -q -r requirements/ci.txt --exists-action w 39 | 40 | test: test.requirements 41 | pytest --cov=xqueue_watcher --cov-report=xml tests 42 | 43 | clean: 44 | find . -name '*.pyc' -delete 45 | 46 | # Targets in a Makefile which do not produce an output file with the same name as the target name 47 | .PHONY: help requirements clean 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ⛔️ WARNING 2 | ========== 3 | 4 | This repository is under-maintained. We are not fixing bugs or developing new features for it. We hope to deprecate and replace it soon. For updates, [follow along on the DEPR ticket](https://github.com/openedx/public-engineering/issues/22) 5 | 6 | Although we have stopped integrating new contributions, we always appreciate security disclosures and patches sent to security@openedx.org 7 | 8 | xqueue_watcher 9 | ========== 10 | 11 | This is an implementation of a polling [XQueue](https://github.com/openedx/xqueue) client and grader. 12 | 13 | Overview 14 | ======== 15 | 16 | There are several components in a working XQueue Watcher service: 17 | - **XQueue Watcher**: it polls an xqueue service continually for new submissions and grades them. 18 | - **Submissions Handler**: when the watcher finds any new submission, it will be passed to the handler for grading. It is a generic handler that can be configured to work with different submissions through individual submission graders. 19 | - **Individual Submission Grader**: each exercise or homework may specify its own "grader". This should map to a file on the server that usually specifies test cases or additional processing for the student submission. 20 | 21 | Usually your server will look like this: 22 | ``` 23 | root/ 24 | ├── xqueue-watcher/ 25 | │ ├── ... # xqueue-watcher repo, unchanged 26 | │ └── ... 27 | ├── config/ 28 | │ └── conf.d/ 29 | │ │ └── my-course.json 30 | │ └── logging.json 31 | └── my-course/ 32 | ├── exercise1/ 33 | │ ├── grader.py # - per-exercise grader 34 | │ └── answer.py # - if using JailedGrader 35 | ├── ... 36 | └── exercise2/ 37 | ├── grader.py 38 | └── answer.py 39 | ``` 40 | Running XQueue Watcher: 41 | ====================== 42 | 43 | Usually you can run XQueue Watcher without making any changes. You should keep course-specific files in another folder like shown above, so that you can update xqueue_watcher anytime. 44 | 45 | Install the requirements before running `xqueue_watcher` 46 | ```bash 47 | cd xqueue-watcher/ 48 | make requirements 49 | ``` 50 | 51 | Now you're ready to run it. 52 | ```bash 53 | python -m xqueue_watcher -d [path to the config directory, eg ../config] 54 | ``` 55 | 56 | The course configuration JSON file in `conf.d` should have the following structure: 57 | ```json 58 | { 59 | "test-123": { 60 | "SERVER": "http://127.0.0.1:18040", 61 | "CONNECTIONS": 1, 62 | "AUTH": ["uname", "pwd"], 63 | "HANDLERS": [ 64 | { 65 | "HANDLER": "xqueue_watcher.grader.Grader", 66 | "KWARGS": { 67 | "grader_root": "/path/to/course/graders/", 68 | } 69 | } 70 | ] 71 | } 72 | } 73 | ``` 74 | 75 | * `test-123`: the name of the queue 76 | * `SERVER`: XQueue server address 77 | * `AUTH`: List containing [username, password] of XQueue Django user 78 | * `CONNECTIONS`: how many threads to spawn to watch the queue 79 | * `HANDLERS`: list of callables that will be called for each queue submission 80 | * `HANDLER`: callable name, see below for Submissions Handler 81 | * `KWARGS`: optional keyword arguments to apply during instantiation 82 | * `grader_root`: path to the course directory, eg /path/to/my-course 83 | 84 | > TODO: document logging.json 85 | 86 | Submissions Handler 87 | =================== 88 | 89 | When `xqueue_watcher` detects any new submission, it will be passed to the submission handler for grading. It will instantiate a new handler based on the name configured above, with submission information retrieved 90 | from XQueue. Base graders are defined in `xqueue_watcher`: Grader and JailedGrader (for Python, using CodeJail). If you don't use JailedGrader, you'd have to implement your own Grader by subclassing `xqueue_watcher.grader.Grader` 91 | 92 | The payload received from XQueue will be a JSON object that usually looks like the JSON below. Note that "grader" is a required field in the "grader_payload" and must be configured accordingly in Studio. 93 | 94 | ```json 95 | { 96 | "student_info": { 97 | "random_seed": 1, 98 | "submission_time": "20210109222647", 99 | "anonymous_student_id": "6d07814a4ece5cdda54af1558a6dfec0" 100 | }, 101 | "grader_payload": "\n {\"grader\": \"relative/path/to/grader.py\"}\n ", 102 | "student_response": "print \"hello\"\r\n " 103 | } 104 | ``` 105 | 106 | ## Custom Handler 107 | To implement a pull grader: 108 | 109 | Subclass `xqueue_watcher.grader.Grader` and override the `grade` method. Then add your grader to the config like `"handler": "my_module.MyGrader"`. The arguments for the `grade` method are: 110 | * `grader_path`: absolute path to the grader defined for the current problem. 111 | * `grader_config`: other configuration particular to the problem 112 | * `student_response`: student-supplied code 113 | 114 | Note that `grader_path` is constructed by appending the relative path to the grader from `grader_payload` to the `grader_root` in the configuration JSON. If the handler cannot find a `grader.py` file, it would fail to grade the submission. 115 | 116 | ## Grading Python submissions with JailedGrader 117 | 118 | `xqueue_watcher` provides a few utilities for grading python submissions, including JailedGrader for running python code in a safe environment and grading support utilities. 119 | 120 | ### JailedGrader 121 | To sandbox python, use [CodeJail](https://github.com/openedx/codejail). In your handler configuration, add: 122 | ```json 123 | "HANDLER": "xqueue_watcher.jailedgrader.JailedGrader", 124 | "CODEJAIL": { 125 | "name": "python", 126 | "python_bin": "/path/to/sandbox/python", 127 | "user": "sandbox_username" 128 | } 129 | ``` 130 | Then, `codejail_python` will automatically be added to the kwargs for your handler. You can then import codejail.jail_code and run `jail_code("python", code...)`. You can define multiple sandboxes and use them as in `jail_code("another-python-version", ...)` 131 | 132 | To use JailedGrader, you also need to provide an `answer.py` file on the same folder with the `grader.py` file. `answer.py` contains the correct/reference implementation of the solution to the problem. The grader will run both student submission and `answer.py` and compare the output with each other. 133 | 134 | ### Grading Support utilities 135 | There are several grading support utilities that make writing `grader.py` for python code easy. Check out 136 | `grader_support/gradelib.py` for the documentation. 137 | 138 | - `grader_support.gradelib.Grader`: a base class for creating a new submission grader. Not to be confused with `xqueue-watcher.grader.Grader`. You can add input checks, preprocessors and tests to a Grader object. 139 | - `grader_support.gradelib.Test`: a base class for creating tests for a submission. Usually a submission can be graded with one or a few tests. There are also few useful test functions and classes included, like `InvokeStudentFunctionTest` , `exec_wrapped_code`, etc. 140 | - Preprocessors: utilities to process the raw submission before grading it. `wrap_in_string` is useful for testing code that is not wrapped in a function. 141 | - Input checks: sanity checks before running a submission, eg check `required_string` or `prohibited_string` 142 | 143 | Using the provided grader class, your `grader.py` would look something like this: 144 | ```python 145 | from grader_support import gradelib 146 | grader = gradelib.Grader() 147 | 148 | # invoke student function foo with parameter [] 149 | grader.add_test(gradelib.InvokeStudentFunctionTest('foo', [])) 150 | ``` 151 | 152 | Or with a pre-processor: 153 | ```python 154 | import gradelib 155 | 156 | grader = gradelib.Grader() 157 | 158 | # execute a raw student code & capture stdout 159 | grader.add_preprocessor(gradelib.wrap_in_string) 160 | grader.add_test(gradelib.ExecWrappedStudentCodeTest({}, "basic test")) 161 | ``` 162 | 163 | You can also write your own test class, processor and input checks. 164 | -------------------------------------------------------------------------------- /conf.d/600.json: -------------------------------------------------------------------------------- 1 | { 2 | "test-123": { 3 | "SERVER": "http://127.0.0.1:18040", 4 | "CONNECTIONS": 1, 5 | "AUTH": ["lms", "lms"], 6 | "HANDLERS": [ 7 | { 8 | "HANDLER": "xqueue_watcher.grader.Grader", 9 | "KWARGS": { 10 | "grader_root": "../data/6.00x/graders/", 11 | "gradepy": "../data/6.00x/graders/grade.py" 12 | } 13 | } 14 | ] 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /conf.d/logging.json: -------------------------------------------------------------------------------- 1 | { 2 | "loggers": { 3 | "": { 4 | "handlers": ["console"], 5 | "level": "DEBUG" 6 | }, 7 | "requests": { 8 | "level": "ERROR", 9 | "handlers": ["console"] 10 | } 11 | }, 12 | "version": 1, 13 | "handlers": { 14 | "console": { 15 | "class": "logging.StreamHandler", 16 | "level": "DEBUG", 17 | "formatter": "standard" 18 | } 19 | }, 20 | "formatters": { 21 | "raw": { 22 | "format": "%(message)s" 23 | }, 24 | "standard": { 25 | "format": "%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s" 26 | } 27 | } 28 | } 29 | 30 | -------------------------------------------------------------------------------- /grader_support/__init__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from . import gradelib, graderutil 3 | # for backwards compatibility, insert gradelib and graderutil 4 | # in the top level 5 | sys.modules['gradelib'] = gradelib 6 | sys.modules['graderutil'] = graderutil 7 | -------------------------------------------------------------------------------- /grader_support/gradelib.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import inspect 3 | import random 4 | import re 5 | import sys 6 | from tokenize import tokenize, COMMENT, STRING 7 | from io import BytesIO, StringIO 8 | 9 | # the run library should overwrite this with a particular random seed for the test. 10 | rand = random.Random(1) 11 | 12 | 13 | class EndTest(Exception): 14 | """ 15 | An exception raised in a test to end the test. 16 | """ 17 | def __init__(self, message=None): 18 | Exception.__init__(self, message) 19 | 20 | 21 | class Test: 22 | """ 23 | A simple class to wrap a test function and its descriptions. 24 | 25 | A test function takes the submission module (the module object 26 | resulting from running the student's code), and prints text. 27 | The text output will be compared to the official answer's output 28 | from the test function to decide if the answer is right. 29 | 30 | """ 31 | def __init__(self, test_fn, short_description, detailed_description='', compare=None): 32 | """ 33 | test_fn: function that takes a submission module and prints something to stdout. 34 | short_description: short description of the test. 35 | detailed_description: (optional) longer description. 36 | """ 37 | self._test_fn = test_fn 38 | self.short_description = short_description 39 | self.detailed_description = detailed_description 40 | if compare: 41 | self.compare_results = compare 42 | 43 | def __call__(self, *args): 44 | return self._test_fn(*args) 45 | 46 | def compare_results(self, expected, actual): 47 | """ 48 | Called to compare the results from the official answer to the student's 49 | results. Return True for pass, False for fail. 50 | 51 | Note: This runs outside the sandbox. 52 | """ 53 | return expected == actual 54 | 55 | 56 | class Grader: 57 | def __init__(self): 58 | """ 59 | Create an empty grader. 60 | 61 | Has the fix_line_endings preprocessor installed by default. 62 | """ 63 | # list of Test objects: 64 | # test(submission_module) -> ??? 65 | # test.short_description : string, should fit on one short line in browser 66 | # test.detailed_description : string, can be '' if no more description is needed. 67 | self._tests = [] 68 | 69 | # list of functions: submission_text -> error text or None 70 | self._input_checks = [] 71 | 72 | # list of functions: submission_text -> processed_submission_text. Run 73 | # in the specified order. (foldl) 74 | self._preprocessors = [fix_line_endings] 75 | 76 | # how many EndTests did we raise and not catch? So we can make sure the student 77 | # didn't catch them. 78 | self._end_tests = 0 79 | 80 | ### Grader interface ############################################################# 81 | def input_errors(self, submission_str): 82 | """ 83 | submission: string 84 | 85 | returns: list of problems / errors that prevent code from being run. If no errors, []. 86 | 87 | MUST NOT RUN the submission. Only allowed to do safe checks, like substr, etc. 88 | """ 89 | return [_f for _f in [check(submission_str) for check in self._input_checks] if _f] 90 | 91 | def preprocess(self, submission_str): 92 | """ 93 | submission: string 94 | 95 | returns: string to actually evaluate. 96 | 97 | MUST NOT RUN the submission. Just add extra stuff, e.g. add a preamble. 98 | """ 99 | s = submission_str 100 | for f in self._preprocessors: 101 | s = f(s) 102 | 103 | return s 104 | 105 | def tests(self): 106 | return self._tests 107 | 108 | def end_test(self, message): 109 | """ 110 | End the test with a message to the student. 111 | """ 112 | self._end_tests += 1 113 | print(_("*** Error: {0}").format(message)) 114 | raise EndTest() 115 | 116 | def caught_end_test(self): 117 | self._end_tests -= 1 118 | 119 | def uncaught_end_tests(self): 120 | """ 121 | How many EndTest exceptions were raised but not caught? 122 | """ 123 | return self._end_tests 124 | 125 | ### Grader setup ############################################################### 126 | 127 | def add_preprocessor(self, fn): 128 | """ 129 | Append preprocessor function to the preprocessors list. It runs after 130 | all the existing preprocessors. 131 | """ 132 | self._preprocessors.append(fn) 133 | 134 | def add_test(self, test): 135 | """ 136 | Append test object to the tests list. 137 | 138 | test is an annotated callable (e.g. a Test object): 139 | test(submission_module) -> anything, printing output to be compared to staff solution 140 | test.short_description : string, should fit on one short line in browser 141 | test.detailed_description : string, can be '' if no more description is needed. 142 | """ 143 | self._tests.append(test) 144 | 145 | def add_tests_from_class(self, test_class): 146 | """ 147 | Add a number of tests, one for each method in a test class. Like this: 148 | 149 | class Tests(object): 150 | def test_pop_from_empty(self, submission_module): 151 | '''q = Queue() 152 | q.remove()''' 153 | q = submission_module.Queue() 154 | try: 155 | q.remove() 156 | print "It should have raised ValueError!" 157 | except ValueError: 158 | print "It did raise ValueError!" 159 | def compare_results(self, expected, actual): 160 | # optional ,if you want a custom grader 161 | 162 | grader.add_tests_from_class(Tests) 163 | 164 | Each method becomes a Test. The name of the test is the short description 165 | (_ are replaced by spaces); any docstring will be the long description. 166 | """ 167 | t = test_class() 168 | if hasattr(t, "compare_results"): 169 | compare = t.compare_results 170 | else: 171 | compare = None 172 | for name, value in inspect.getmembers(t, inspect.ismethod): 173 | if name.startswith("test_"): 174 | sd = 'Test: ' + name[5:].replace('_', ' ') 175 | self.add_test(Test(value, sd, value.__doc__, compare)) 176 | 177 | 178 | def add_input_check(self, check): 179 | """ 180 | Add an input check function to the grader. 181 | 182 | check is a function: student submission string -> complaint str, or None if ok. 183 | MUST NOT run the submission. 184 | """ 185 | self._input_checks.append(check) 186 | 187 | 188 | ## Preprocessors ########################################################## 189 | 190 | def wrap_in_string(code): 191 | """ 192 | For testing code that's not wrapped in a function (e.g. only for the first few problems), 193 | wrap it in a string, so that tests can exec it multiple times. 194 | 195 | See also exec_wrapped_code() below. 196 | """ 197 | # repr() takes care of escaping things properly. 198 | s = "submission_code = " + repr(code) 199 | #print "wrapped '" + code + "': '" + s + "'" 200 | 201 | return s 202 | 203 | def fix_line_endings(code): 204 | """ 205 | Remove carriage returns. 206 | """ 207 | return code.replace('\r', '') 208 | 209 | ## Input checks ########################################################### 210 | 211 | def required_substring(string, error_msg=None): 212 | """ 213 | Returns a function that checks that string is present in the code, returning 214 | error_msg if it isn't there. If error_msg is None, returns a default error message. 215 | """ 216 | def check(code): 217 | if code.find(string) == -1: 218 | return error_msg or _("Your code should contain '{0}'.").format(string) 219 | return None 220 | 221 | return check 222 | 223 | def prohibited_substring(string, error_msg=None): 224 | """ 225 | Returns a function that checks that string is not present in the code, returning 226 | error_msg if it is present. If error_msg is None, returns a default error message. 227 | """ 228 | def check(code): 229 | if code.find(string) != -1: 230 | return error_msg or _("Your code should not contain '{0}'.").format(string) 231 | return None 232 | 233 | return check 234 | 235 | # Aliases that might become more intelligent in the future. 236 | def _tokens(code): 237 | """ 238 | A wrapper around tokenize.generate_tokens. 239 | """ 240 | # Protect against pathological inputs: http://bugs.python.org/issue16152 241 | toks = tokenize(BytesIO(code.encode('utf-8')).readline) 242 | return toks 243 | 244 | def _count_tokens(code, string): 245 | """ 246 | Return a count of how many times `string` appears as a keyword in `code`. 247 | """ 248 | count = 0 249 | 250 | try: 251 | for ttyp, ttok, __, __, __ in _tokens(code): 252 | if ttyp in (COMMENT, STRING): 253 | continue 254 | if ttok == string: 255 | count += 1 256 | except: 257 | # The input code was bad in some way. It will fail later on. 258 | pass 259 | return count 260 | 261 | def prohibited_keyword(string, error_msg=None): 262 | def check(code): 263 | if _count_tokens(code, string) > 0: 264 | return error_msg or _('Your code cannot make use of the "{0}" keyword.').format(string) 265 | return None 266 | 267 | return check 268 | 269 | def required_keyword(string, error_msg=None): 270 | def check(code): 271 | if _count_tokens(code, string) == 0: 272 | return error_msg or _('Your code must make use of the "{0}" keyword.').format(string) 273 | return None 274 | 275 | return check 276 | 277 | def input_check_or(error_msg, *args): 278 | def check(code): 279 | for check in args: 280 | if check(code) is None: 281 | return None 282 | return error_msg 283 | return check 284 | 285 | def one_of_required_keywords(strings, error_msg=None): 286 | """ 287 | strings is a list of strings 288 | returns True if one of the strings is present, False if none are. 289 | """ 290 | def check(code): 291 | for string in strings: 292 | if _count_tokens(code, string) > 0: 293 | return None 294 | 295 | return error_msg or _('Your code must make use of at least one of the following keywords: {0}.').format(strings) 296 | 297 | return check 298 | 299 | prohibited_operator = prohibited_keyword 300 | required_operator = required_keyword 301 | 302 | def substring_occurs(string, at_least=None, at_most=None, exactly=None, error_msg=None, ignore_spacing=False): 303 | """ 304 | Returns an input check function that checks that `string` occurs at least 305 | `at_least` times, and/or not more than `at_most` times, or exactly `exactly` 306 | times. 307 | """ 308 | def check(code): 309 | if ignore_spacing: 310 | occurs = code.replace(' ', '').count(string.replace(' ', '')) 311 | else: 312 | occurs = code.count(string) 313 | return _check_occurs(string, occurs, at_least, at_most, exactly, error_msg) 314 | 315 | return check 316 | 317 | def substring_occurs_if_condstring(string, condstring, at_least=None, at_most=None, exactly=None, error_msg=None): 318 | """ 319 | Returns an input check function that checks that `string` occurs at least 320 | `at_least` times, and/or not more than `at_most` times, or exactly `exactly` 321 | times, if `condstring` occurs at least once in the string. 322 | """ 323 | def check(code): 324 | condoccurs = code.count(condstring) 325 | if condoccurs: 326 | occurs = code.count(string) 327 | return _check_occurs(string, occurs, at_least, at_most, exactly, error_msg) 328 | 329 | return None 330 | 331 | return check 332 | 333 | def token_occurs(string, at_least=None, at_most=None, exactly=None, error_msg=None): 334 | """ 335 | Returns an input check function that checks that `string` occurs at least 336 | `at_least` times, and/or not more than `at_most` times, or exactly `exactly` 337 | times. Only occurrences outside of strings and comments are counted. 338 | """ 339 | def check(code): 340 | occurs = _count_tokens(code, string) 341 | return _check_occurs(string, occurs, at_least, at_most, exactly, error_msg) 342 | return check 343 | 344 | def count_non_comment_lines(at_least=None, at_most=None, exactly=None, error_msg=None): 345 | """ 346 | Returns an input check function that checks that the number of non-comment, 347 | non-blank source lines conforms to the rules in the arguments. 348 | """ 349 | def check(code): 350 | linenums = set() 351 | for ttyp, ttok, (srow, __), __, __ in _tokens(code): 352 | if ttyp in (COMMENT, STRING): 353 | # Comments and strings don't count toward line count. If a string 354 | # is the only thing on a line, then it's probably a docstring, so 355 | # don't count it. 356 | continue 357 | if not ttok.strip(): 358 | # Tokens that are only whitespace don't count. 359 | continue 360 | linenums.add(srow) 361 | num = len(linenums) 362 | return _check_occurs(None, num, at_least, at_most, exactly, error_msg) 363 | return check 364 | 365 | def _check_occurs(text, occurs, at_least=None, at_most=None, exactly=None, error_msg=None): 366 | if exactly is not None: 367 | if occurs != exactly: 368 | return error_msg or _("Your code has {0!r} {1} times, must be exactly {2}.").format(text, occurs, exactly) 369 | if at_least is not None: 370 | if occurs < at_least: 371 | return error_msg or _("Your code has {0!r} {1} times, must be at least {2}.").format(text, occurs, at_least) 372 | if at_most is not None: 373 | if occurs > at_most: 374 | return error_msg or _("Your code has {0!r} {1} times, can't be more than {2}.").format(text, occurs, at_most) 375 | return None 376 | 377 | def must_define_function(fn_name, error_msg=None): 378 | """ 379 | Returns a function that checks if a function named `fn_name` is defined. If not, 380 | returns `error_msg`, or a default message. 381 | """ 382 | def check(code): 383 | if not re.search(r"\bdef\s+%s\b" % fn_name, code): 384 | return error_msg or _("Your code must define a function named '{0}'.").format(fn_name) 385 | return None 386 | 387 | return check 388 | 389 | def prohibited_function_definition(fn_name, error_msg=None): 390 | """ 391 | Returns a function that checks if a function named `fn_name` is defined. If so, 392 | returns `error_msg`, or a default message. 393 | """ 394 | def check(code): 395 | if re.search(r"\bdef\s+%s\b" % fn_name, code): 396 | return error_msg or _("Your code should NOT define a function named '{0}'.").format(fn_name) 397 | return None 398 | 399 | return check 400 | 401 | def must_define_class(class_name, error_msg=None): 402 | """ 403 | Returns a function that checks if a class named `class_name` is defined. If not, 404 | returns `error_msg`, or a default message. 405 | """ 406 | def check(code): 407 | # if not re.search(r"\bclass\s+%s\b" % class_name, code): 408 | if 'class ' + class_name not in code: 409 | return error_msg or _("Your code must define a class named '{0}'. Be sure you only have one space between the keyword 'class' and the class name.").format(class_name) 410 | return None 411 | 412 | return check 413 | 414 | def prohibited_class_method(class_name, method_name, error_msg=None): 415 | """ 416 | Returns a function that checks if a class named `class_name` contains a method 417 | titled `method_name`. If so, returns `error_msg`, or a default message. 418 | """ 419 | def check(code): 420 | in_class = False 421 | lines = code.split('\n') 422 | # Remove comments from lines 423 | lines = [line[:line.find('#')] for line in lines] 424 | for line in lines: 425 | if line.replace(' ', '') == '': 426 | continue 427 | if 'class ' + class_name in line: 428 | in_class = True 429 | elif in_class and re.search(r"\bdef\s+%s\b" % method_name, line): 430 | return error_msg or _("The class named '{0}' should not define a method named {1}.").format(class_name, method_name) 431 | elif in_class and 'class ' in line: 432 | if class_name not in line or '('+class_name+')' in line.replace(' ', ''): 433 | in_class = False 434 | return None 435 | return check 436 | 437 | def required_class_method(class_name, method_name, error_msg=None): 438 | """ 439 | Returns a function that checks if a class named `class_name` contains a method 440 | titled `method_name`. If not, returns `error_msg`, or a default message. 441 | """ 442 | def check(code): 443 | in_class = False 444 | lines = code.split('\n') 445 | # Remove comments from lines 446 | lines = [line[:line.find('#')] for line in lines] 447 | for line in lines: 448 | if line.replace(' ', '') == '': 449 | continue 450 | if 'class ' + class_name in line: 451 | in_class = True 452 | elif in_class and re.search(r"\bdef\s+%s\b" % method_name, line): 453 | return None 454 | elif in_class and 'class ' in line and class_name not in line: 455 | in_class = False 456 | return error_msg or _("The class named '{0}' should define a method named {1}.").format(class_name, method_name) 457 | return check 458 | 459 | 460 | ## test functions ################################# 461 | 462 | @contextlib.contextmanager 463 | def capture_stdout(): 464 | old_stdout = sys.stdout 465 | sys.stdout = stdout = StringIO() 466 | yield stdout 467 | sys.stdout = old_stdout 468 | 469 | def exec_wrapped_code(environment=None, post_process=None): 470 | """ 471 | Exec the submission code, with the given environment. 472 | `post_process` is a function that takes a string, the original stdout of 473 | the code, and returns a new string, the transformed stdout, suitable for 474 | comparison. 475 | """ 476 | if environment is None: 477 | environment = {} 478 | def test_fn(submission_module): 479 | with capture_stdout() as stdout: 480 | exec(submission_module.submission_code, environment) 481 | stdout_text = stdout.getvalue() 482 | if post_process: 483 | stdout_text = post_process(stdout_text) 484 | print(stdout_text) 485 | 486 | return test_fn 487 | 488 | 489 | def exec_code_and_inspect_values(environment=None, vars_to_inspect=None, post_process=None): 490 | """ 491 | Exec the submission code, with the given environment. 492 | `vars_to_inspect` is a list of variables to inspect the values of (they will 493 | be inspected by printing to stdout). Does not otherwise inspect student output. 494 | `post_process` is a function that takes a string, the original stdout of 495 | the code, and returns a new string, the transformed stdout, suitable for 496 | comparison. 497 | """ 498 | if environment is None: 499 | environment = {} 500 | def test_fn(submission_module): 501 | with capture_stdout() as stdout: 502 | exec(submission_module.submission_code, environment) 503 | 504 | for var in vars_to_inspect: 505 | print(var) 506 | 507 | return test_fn 508 | 509 | 510 | def trace_wrapped_code(inspector, error_msg): 511 | def test_fn(submission_module): 512 | inspector.set_source(submission_module.submission_code) 513 | with inspector: 514 | for report in inspector.inspect_dispatch(): 515 | if not report: 516 | print(error_msg) 517 | return test_fn 518 | 519 | def invoke_student_function(fn_name, args, environment=None, output_writer=None): 520 | """ 521 | Run the student's function named `fn_name` with the args `args`, and prints 522 | the result. `output_writer` is a function that takes the result of the student's 523 | function, and produces a string to print. It defaults to `repr`, but for example, 524 | floats should be formatted to a particular number of decimal places to prevent 525 | rounding issues. 526 | """ 527 | output_writer = output_writer or repr 528 | def doit(submission_module): 529 | for name, value in (environment or {}).items(): 530 | setattr(submission_module, name, value) 531 | fn = getattr(submission_module, fn_name) 532 | print(output_writer(fn(*args))) 533 | return doit 534 | 535 | class InvokeStudentFunctionTest(Test): 536 | """ 537 | A Test that invokes a student function. 538 | """ 539 | def __init__(self, fn_name, args, environment=None, output_writer=None, short_desc=None, detailed_desc=None, compare=None): 540 | test_fn = invoke_student_function(fn_name, args, environment, output_writer) 541 | if short_desc is None: 542 | short_desc = "Test: {}({})".format(fn_name, ", ".join(repr(a) for a in args)) 543 | Test.__init__(self, test_fn, short_desc, detailed_desc, compare) 544 | 545 | class ExecWrappedStudentCodeTest(Test): 546 | """ 547 | A Test that executes the student code and captures the result, which is output to stdout. 548 | The code must be preprocessed with `wrap_in_string` 549 | """ 550 | def __init__(self, environment=None, short_desc=None, detailed_desc=None, compare=None): 551 | test_fn = exec_wrapped_code(environment) 552 | if short_desc is None: 553 | short_desc = "Test: %s(%s)" % (fn_name, ", ".join(repr(a) for a in args)) 554 | gradelib.Test.__init__(self, test_fn, short_desc, detailed_desc, compare) 555 | 556 | def round_float_writer(n): 557 | """ 558 | Returns an output_writer function that rounds its argument to `n` places. 559 | """ 560 | def _round_float_output_writer(f): 561 | return "%.*f" % (n, f) 562 | return _round_float_output_writer 563 | -------------------------------------------------------------------------------- /grader_support/graderutil.py: -------------------------------------------------------------------------------- 1 | """ 2 | Utilities to help manage code execution and testing. 3 | """ 4 | 5 | import contextlib 6 | import os, os.path 7 | import shutil 8 | import sys 9 | import tempfile 10 | import textwrap 11 | import traceback 12 | 13 | import io 14 | 15 | # Set this variable to the language code you wish to use 16 | # Default is 'en'. Dummy translations are served up in 'eo'. 17 | # Put your translations in the file `graders/conf/locale/LANGUAGE/LC_MESSAGES/graders.mo` 18 | LANGUAGE = 'en' 19 | 20 | 21 | @contextlib.contextmanager 22 | def captured_stdout(): 23 | """ 24 | A context manager to capture stdout into a StringIO. 25 | 26 | with captured_stdout() as stdout: 27 | # .. print stuff .. 28 | stdout.getvalue() # this is a string with what got printed. 29 | 30 | """ 31 | old_stdout = sys.stdout 32 | sys.stdout = stdout = io.StringIO() 33 | 34 | try: 35 | yield stdout 36 | finally: 37 | sys.stdout = old_stdout 38 | 39 | 40 | class ChangeDirectory: 41 | def __init__(self, new_dir): 42 | self.old_dir = os.getcwd() 43 | os.chdir(new_dir) 44 | 45 | def clean_up(self): 46 | os.chdir(self.old_dir) 47 | 48 | 49 | @contextlib.contextmanager 50 | def change_directory(new_dir): 51 | """ 52 | A context manager to change the directory, and then change it back. 53 | """ 54 | cd = ChangeDirectory(new_dir) 55 | try: 56 | yield new_dir 57 | finally: 58 | cd.clean_up() 59 | 60 | 61 | class TempDirectory: 62 | def __init__(self, delete_when_done=True): 63 | self.delete_when_done = delete_when_done 64 | self.temp_dir = tempfile.mkdtemp(prefix="grader-") 65 | # Make directory readable by other users ('sandbox' user needs to be able to read it) 66 | os.chmod(self.temp_dir, 0o775) 67 | 68 | def clean_up(self): 69 | if self.delete_when_done: 70 | # if this errors, something is genuinely wrong, so don't ignore errors. 71 | shutil.rmtree(self.temp_dir) 72 | 73 | 74 | @contextlib.contextmanager 75 | def temp_directory(delete_when_done=True): 76 | """ 77 | A context manager to make and use a temp directory. If `delete_when_done` 78 | is true (the default), the directory will be removed when done. 79 | """ 80 | tmp = TempDirectory(delete_when_done) 81 | try: 82 | yield tmp.temp_dir 83 | finally: 84 | tmp.clean_up() 85 | 86 | 87 | class ModuleIsolation: 88 | """ 89 | Manage changes to sys.modules so that we can roll back imported modules. 90 | 91 | Create this object, it will snapshot the currently imported modules. When 92 | you call `clean_up()`, it will delete any module imported since its creation. 93 | """ 94 | 95 | def __init__(self): 96 | # Save all the names of all the imported modules. 97 | self.mods = set(sys.modules) 98 | 99 | def clean_up(self): 100 | # Get a list of modules that didn't exist when we were created 101 | new_mods = [m for m in sys.modules if m not in self.mods] 102 | # and delete them all so another import will run code for real again. 103 | for m in new_mods: 104 | del sys.modules[m] 105 | 106 | 107 | @contextlib.contextmanager 108 | def module_isolation(): 109 | mi = ModuleIsolation() 110 | try: 111 | yield 112 | finally: 113 | mi.clean_up() 114 | 115 | 116 | def make_file(filename, text=""): 117 | """Create a file. 118 | 119 | `filename` is the path to the file, including directories if desired, 120 | and `text` is the content. 121 | 122 | Returns the path to the file. 123 | 124 | """ 125 | # Make sure the directories are available. 126 | dirs, __ = os.path.split(filename) 127 | if dirs and not os.path.exists(dirs): 128 | os.makedirs(dirs) 129 | 130 | # Create the file. 131 | with open(filename, 'wb') as f: 132 | f.write(textwrap.dedent(text)) 133 | 134 | return filename 135 | 136 | 137 | def format_exception(exc_info=None, main_file=None, hide_file=False): 138 | """ 139 | Format an exception, defaulting to the currently-handled exception. 140 | `main_file` is the filename that should appear as the top-most frame, 141 | to hide the context in which the code was run. If `hide_file` is true, 142 | then file names in the stack trace are made relative to the current 143 | directory. 144 | """ 145 | exc_info = exc_info or sys.exc_info() 146 | exc_type, exc_value, exc_tb = exc_info 147 | if main_file: 148 | while exc_tb is not None and not frame_in_file(exc_tb.tb_frame, main_file): 149 | exc_tb = exc_tb.tb_next 150 | lines = traceback.format_exception(exc_type, exc_value, exc_tb) 151 | if hide_file: 152 | cwd = os.getcwd() + os.sep 153 | lines = [l.replace(cwd, "", 1) for l in lines] 154 | return "".join(lines) 155 | 156 | 157 | def frame_in_file(frame, filename): 158 | """ 159 | Does the traceback frame `frame` reference code in `filename`? 160 | """ 161 | frame_file = frame.f_code.co_filename 162 | frame_stem = os.path.splitext(os.path.basename(frame_file))[0] 163 | filename_stem = os.path.splitext(filename)[0] 164 | return frame_stem == filename_stem 165 | -------------------------------------------------------------------------------- /grader_support/run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Run a set of tests on a submission, printing the outputs to stdout as a json 4 | string. 5 | 6 | Note: this command will run student code, so should be run in a sandbox, or 7 | (and!) the code should be sanitized first. Because this just runs the code on 8 | various sample inputs, and does not have answers, bad student code can only hurt 9 | itself. 10 | """ 11 | 12 | import gettext 13 | import json 14 | import random 15 | import sys 16 | 17 | from . import gradelib # to set the random seed 18 | from . import graderutil 19 | 20 | usage = "Usage: run.py GRADER SUBMISSION seed" # pylint: disable=invalid-name 21 | 22 | # Install gettext for translation support. This gettext install works within the sandbox, 23 | # so the path to graders/conf/locale can be relative. 24 | # LANGUAGE is set in graderutil.py 25 | trans = gettext.translation( # pylint: disable=invalid-name 26 | 'graders', 27 | localedir='conf/locale', 28 | fallback=True, 29 | languages=[graderutil.LANGUAGE] 30 | ) 31 | _ = trans.gettext 32 | trans.install(names=None) 33 | 34 | 35 | def run(grader_name, submission_name, seed=1): 36 | """ 37 | `grader_name`: importable module name of the grader 38 | `submission_name`: importable module name of the submission 39 | `seed`: A value to seed randomness with. 40 | 41 | Returns a data structure: 42 | 43 | { 44 | 'grader': { 45 | 'status': 'ok', # or 'error', 'nograder' 46 | 'stdout': 'whatever grader printed', 47 | 'exception': 'a stack trace if error' 48 | }, 49 | 'submission': { 50 | 'status': 'ok', # or 'error', 'caught' 51 | 'stdout': 'whatever the submission printed', 52 | 'exception': 'a stack trace if error' 53 | }, 54 | 'results': [ 55 | ["Test short desc", "test detailed description", "test output..."], 56 | ... 57 | ], 58 | 'exceptions': 0, # or however many were caught. 59 | } 60 | 61 | """ 62 | 63 | output = { 64 | 'grader': { 65 | 'status': 'notrun', 66 | }, 67 | 'submission': { 68 | 'status': 'notrun', 69 | }, 70 | 'results': [], 71 | 'exceptions': 0, 72 | } 73 | 74 | # Use a private random number generator, so student code won't accidentally 75 | # mess it up. (if they mess it up deliberately, we don't care--it only 76 | # hurts them). 77 | gradelib.rand = random.Random(seed) 78 | # Also seed the random singleton in case the exercise uses random numbers. 79 | random.seed(seed + 1) 80 | 81 | grader_mod, results = import_captured(grader_name, our_code=True) 82 | if grader_mod: 83 | try: 84 | grader = grader_mod.grader 85 | except: # pylint: disable=bare-except 86 | results['status'] = 'error' 87 | results['exception'] = graderutil.format_exception() 88 | output['exceptions'] += 1 89 | else: 90 | output['exceptions'] += 1 91 | output['grader'].update(results) 92 | 93 | if output['grader']['status'] == 'ok': 94 | submission, results = import_captured(submission_name) 95 | output['submission'].update(results) 96 | 97 | if submission and output['submission']['status'] == 'ok': 98 | # results is a list of ("short description", "detailed desc", "output") tuples. 99 | try: 100 | for test in grader.tests(): 101 | with graderutil.captured_stdout() as test_stdout: 102 | try: 103 | exception_output = "" 104 | test(submission) 105 | except gradelib.EndTest: 106 | grader.caught_end_test() 107 | except: # pylint: disable=bare-except 108 | # The error could be either the grader code or the submission code, 109 | # so hide information. 110 | exception_output = graderutil.format_exception( 111 | main_file=submission_name, 112 | hide_file=True 113 | ) 114 | output['exceptions'] += 1 115 | else: 116 | exception_output = "" 117 | # Get the output, including anything printed, and any exception. 118 | test_output = test_stdout.getvalue() 119 | if test_output and test_output[-1] != '\n': 120 | test_output += '\n' 121 | test_output += exception_output 122 | output['results'].append( 123 | (test.short_description, test.detailed_description, test_output) 124 | ) 125 | except: # pylint: disable=bare-except 126 | output['grader']['status'] = 'error' 127 | output['grader']['exception'] = graderutil.format_exception() 128 | output['exceptions'] += 1 129 | else: 130 | output['exceptions'] += 1 131 | 132 | if grader.uncaught_end_tests(): 133 | # We raised EndTest more than we caught them, the student must be 134 | # catching them, inadvertently or not. 135 | output['submission']['exception'] = _( 136 | "Your code interfered with our grader. Don't use bare 'except' clauses.") # pylint: disable=line-too-long 137 | output['submission']['status'] = 'caught' 138 | return output 139 | 140 | 141 | def import_captured(name, our_code=False): 142 | """ 143 | Import the module `name`, capturing stdout, and any exceptions that happen. 144 | Returns the module, and a dict of results. 145 | 146 | If `our_code` is true, then the code is edX-authored, and any exception output 147 | can include full context. If `our_code` is false, then this is student-submitted 148 | code, and should have only student-provided information visible in exception 149 | traces. This isn't a security precaution, it just keeps us from showing confusing 150 | and unhelpful information to students. 151 | """ 152 | result = { 153 | 'status': 'notrun', 154 | } 155 | try: 156 | with graderutil.captured_stdout() as stdout: 157 | mod = __import__(name) 158 | except: # pylint: disable=bare-except 159 | result['status'] = 'error' 160 | if our_code: 161 | exc = graderutil.format_exception() 162 | else: 163 | exc = graderutil.format_exception(main_file=name, hide_file=True) 164 | result['exception'] = exc 165 | mod = None 166 | else: 167 | result['status'] = 'ok' 168 | result['stdout'] = stdout.getvalue() 169 | return mod, result 170 | 171 | 172 | def main(args): # pragma: no cover 173 | """ 174 | Execute the grader from the command line 175 | """ 176 | if len(args) != 3: 177 | print(usage) 178 | return 179 | 180 | (grader_path, submission_path, seed) = args 181 | seed = int(seed) 182 | 183 | # strip off .py 184 | grader_name = grader_path[:-3] 185 | submission_name = submission_path[:-3] 186 | 187 | output = run(grader_name, submission_name, seed) 188 | print(json.dumps(output)) 189 | 190 | 191 | if __name__ == '__main__': # pragma: no cover 192 | main(sys.argv[1:]) 193 | -------------------------------------------------------------------------------- /load_test/mock_xqueue.py: -------------------------------------------------------------------------------- 1 | import flask 2 | import itertools 3 | import random 4 | import time 5 | 6 | app = flask.Flask(__name__) 7 | 8 | counter = itertools.count() 9 | 10 | SUBMISSIONS = [ 11 | ('', 'ps03/derivatives/grade_derivatives.py'), 12 | ('import time;time.sleep(100)', 'ps03/derivatives/grade_derivatives.py'), 13 | (''' 14 | def foo(): 15 | return "hello" 16 | ''', 'ps02/bisect/grade_bisect.py'), 17 | (''' 18 | deff bad(): 19 | haha 20 | ''', 'ps02/bisect/grade_bisect.py'), 21 | (''' 22 | monthlyInterestRate = annualInterestRate/12 23 | lo = balance/12 24 | hi = (balance*(1+monthlyInterestRate)**12)/12 25 | done = False 26 | while not done : 27 | payment = (lo+hi)/2 28 | nb = balance 29 | for m in range(0,12) : 30 | nb = (nb-payment)*(1+monthlyInterestRate) 31 | done = (abs(nb) < .005); 32 | if (nb > 0) : 33 | lo = payment 34 | else : 35 | hi = payment 36 | print('Lowest Payment: %.2f' % payment) 37 | ''', 'ps02/bisect/grade_bisect.py'), 38 | ] 39 | 40 | COUNTERS = { 41 | 'requests': 0, 42 | 'results': 0, 43 | 'start': time.time() 44 | } 45 | 46 | @app.route('/start') 47 | def start(): 48 | COUNTERS['start'] = time.time() 49 | COUNTERS['requests'] = COUNTERS['results'] = 0 50 | return flask.jsonify(COUNTERS) 51 | 52 | @app.route('/stats') 53 | def stats(): 54 | timediff = time.time() - COUNTERS['start'] 55 | response = { 56 | 'requests_per_second': COUNTERS['requests'] / timediff, 57 | 'posts_per_second': COUNTERS['results'] / timediff 58 | } 59 | return flask.jsonify(response) 60 | 61 | 62 | @app.route('/xqueue/get_submission/') 63 | def get_submission(): 64 | idx = random.randint(0, len(SUBMISSIONS) - 1) 65 | submission, grader = SUBMISSIONS[idx] 66 | payload = { 67 | 'grader': grader 68 | } 69 | response = { 70 | 'return_code': 0, 71 | 'content': flask.json.dumps({ 72 | 'xqueue_header': '{}.{}'.format(next(counter), idx), 73 | 'xqueue_body': flask.json.dumps({ 74 | 'student_response': submission, 75 | 'grader_payload': flask.json.dumps(payload) 76 | }), 77 | 'xqueue_files': '' 78 | }) 79 | } 80 | COUNTERS['requests'] += 1 81 | return flask.jsonify(response) 82 | 83 | @app.route('/xqueue/login/', methods=['POST']) 84 | def login(): 85 | return flask.jsonify({'return_code': 0}) 86 | 87 | 88 | @app.route('/xqueue/put_result/', methods=['POST']) 89 | def put_result(): 90 | COUNTERS['results'] += 1 91 | return flask.jsonify({'return_code': 0, 'content': 'thank you'}) 92 | 93 | 94 | 95 | if __name__ == '__main__': 96 | app.run(debug=True) 97 | -------------------------------------------------------------------------------- /load_test/run.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import subprocess 3 | import os 4 | import sys 5 | import time 6 | import requests 7 | import json 8 | import tempfile 9 | import getpass 10 | from path import Path 11 | import pprint 12 | import argparse 13 | 14 | 15 | WATCHER_CONFIG = { 16 | "load-test": { 17 | "SERVER": 'undefined', 18 | "CONNECTIONS": 1, 19 | "HANDLERS": [ 20 | { 21 | "HANDLER": "xqueue_watcher.jailedgrader.JailedGrader", 22 | "KWARGS": { 23 | "grader_root": Path(__file__).dirname() / "../../data/6.00x/graders/", 24 | } 25 | } 26 | ] 27 | } 28 | } 29 | 30 | def start_mock_xqueue(port): 31 | cmd = 'gunicorn -w 1 -k gevent -b 0.0.0.0:%s mock_xqueue:app' % port 32 | print(cmd) 33 | proc = subprocess.Popen(cmd.split()) 34 | return proc 35 | 36 | def start_queue_watcher(config_file, codejail_config_file): 37 | cmd = f'python -m xqueue_watcher -f {config_file} -j {codejail_config_file}' 38 | print(cmd) 39 | proc = subprocess.Popen(cmd.split()) 40 | return proc 41 | 42 | def get_stats(server_address): 43 | pprint.pprint(requests.get('%s/stats' % server_address).json()) 44 | print('\n') 45 | 46 | def main(args): 47 | parser = argparse.ArgumentParser() 48 | parser.add_argument('-c', '--concurrency', default=2, type=int, help='number of watchers') 49 | parser.add_argument('-x', '--xqueue', action="store_true", help='run mock xqueue', default=False) 50 | parser.add_argument('-w', '--watcher', action="store_true", help='run watcher', default=False) 51 | parser.add_argument('-a', '--xqueue-address', help='xqueue address', default='http://127.0.0.1:18042') 52 | args = parser.parse_args(args) 53 | 54 | if not (args.xqueue or args.watcher): 55 | parser.print_help() 56 | return -1 57 | 58 | if args.xqueue: 59 | port = args.xqueue_address.split(':')[-1] 60 | xqueue_proc = start_mock_xqueue(port) 61 | 62 | time.sleep(2) 63 | else: 64 | xqueue_proc = None 65 | 66 | if args.watcher: 67 | codejail_config = tempfile.NamedTemporaryFile(delete=False) 68 | json.dump({'python_bin': sys.executable, 'user': getpass.getuser()}, codejail_config) 69 | codejail_config.close() 70 | 71 | watcher_config = tempfile.NamedTemporaryFile(delete=False) 72 | WATCHER_CONFIG['load-test']['CONNECTIONS'] = args.concurrency 73 | WATCHER_CONFIG['load-test']['SERVER'] = args.xqueue_address 74 | json.dump(WATCHER_CONFIG, watcher_config) 75 | watcher_config.close() 76 | pprint.pprint(WATCHER_CONFIG) 77 | 78 | watcher_proc = start_queue_watcher(watcher_config.name, codejail_config.name) 79 | time.sleep(1) 80 | print(requests.get('%s/start' % args.xqueue_address).json()) 81 | else: 82 | watcher_proc = None 83 | 84 | 85 | while watcher_proc or xqueue_proc: 86 | try: 87 | time.sleep(2) 88 | get_stats(args.xqueue_address) 89 | except KeyboardInterrupt: 90 | break 91 | 92 | if watcher_proc: 93 | os.kill(watcher_proc.pid, 15) 94 | codejail_config.unlink(codejail_config.name) 95 | watcher_config.unlink(watcher_config.name) 96 | 97 | if xqueue_proc: 98 | os.kill(xqueue_proc.pid, 15) 99 | 100 | print('\n\ndone') 101 | 102 | if __name__ == '__main__': 103 | main(sys.argv[1:]) -------------------------------------------------------------------------------- /openedx.yaml: -------------------------------------------------------------------------------- 1 | # This file describes this Open edX repo, as described in OEP-2: 2 | # http://open-edx-proposals.readthedocs.io/en/latest/oeps/oep-0002.html#specification 3 | 4 | nick: xqw 5 | oeps: 6 | oep-7: false 7 | oep-18: true 8 | 9 | tags: 10 | - backend-service 11 | -------------------------------------------------------------------------------- /requirements/base.in: -------------------------------------------------------------------------------- 1 | # Core requirements for using this package 2 | 3 | -c constraints.txt 4 | 5 | dogstatsd-python 6 | path.py 7 | requests 8 | six 9 | 10 | -e git+https://github.com/openedx/codejail.git@4127fc4bd5775cc72aee8d7f0a70e31405e22439#egg=codejail 11 | -------------------------------------------------------------------------------- /requirements/base.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.11 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | -e git+https://github.com/openedx/codejail.git@4127fc4bd5775cc72aee8d7f0a70e31405e22439#egg=codejail 8 | # via -r requirements/base.in 9 | certifi==2025.4.26 10 | # via requests 11 | charset-normalizer==3.4.2 12 | # via requests 13 | dogstatsd-python==0.5.6 14 | # via -r requirements/base.in 15 | idna==3.10 16 | # via requests 17 | path==17.1.0 18 | # via path-py 19 | path-py==12.5.0 20 | # via -r requirements/base.in 21 | requests==2.32.3 22 | # via -r requirements/base.in 23 | six==1.17.0 24 | # via -r requirements/base.in 25 | urllib3==2.2.3 26 | # via 27 | # -c requirements/common_constraints.txt 28 | # requests 29 | -------------------------------------------------------------------------------- /requirements/ci.in: -------------------------------------------------------------------------------- 1 | # Requirements for running tests in CI 2 | -c constraints.txt 3 | 4 | -r test.txt 5 | 6 | coverage 7 | -------------------------------------------------------------------------------- /requirements/ci.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.11 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | -e git+https://github.com/openedx/codejail.git@4127fc4bd5775cc72aee8d7f0a70e31405e22439#egg=codejail 8 | # via -r requirements/test.txt 9 | certifi==2025.4.26 10 | # via 11 | # -r requirements/test.txt 12 | # requests 13 | charset-normalizer==3.4.2 14 | # via 15 | # -r requirements/test.txt 16 | # requests 17 | coverage[toml]==7.8.1 18 | # via 19 | # -r requirements/ci.in 20 | # -r requirements/test.txt 21 | # pytest-cov 22 | dogstatsd-python==0.5.6 23 | # via -r requirements/test.txt 24 | idna==3.10 25 | # via 26 | # -r requirements/test.txt 27 | # requests 28 | iniconfig==2.1.0 29 | # via 30 | # -r requirements/test.txt 31 | # pytest 32 | mock==5.2.0 33 | # via -r requirements/test.txt 34 | packaging==25.0 35 | # via 36 | # -r requirements/test.txt 37 | # pytest 38 | path==17.1.0 39 | # via 40 | # -r requirements/test.txt 41 | # path-py 42 | path-py==12.5.0 43 | # via -r requirements/test.txt 44 | pluggy==1.6.0 45 | # via 46 | # -r requirements/test.txt 47 | # pytest 48 | pytest==8.3.5 49 | # via 50 | # -r requirements/test.txt 51 | # pytest-cov 52 | pytest-cov==6.1.1 53 | # via -r requirements/test.txt 54 | requests==2.32.3 55 | # via -r requirements/test.txt 56 | six==1.17.0 57 | # via -r requirements/test.txt 58 | urllib3==2.2.3 59 | # via 60 | # -c requirements/common_constraints.txt 61 | # -r requirements/test.txt 62 | # requests 63 | -------------------------------------------------------------------------------- /requirements/common_constraints.txt: -------------------------------------------------------------------------------- 1 | # A central location for most common version constraints 2 | # (across edx repos) for pip-installation. 3 | # 4 | # Similar to other constraint files this file doesn't install any packages. 5 | # It specifies version constraints that will be applied if a package is needed. 6 | # When pinning something here, please provide an explanation of why it is a good 7 | # idea to pin this package across all edx repos, Ideally, link to other information 8 | # that will help people in the future to remove the pin when possible. 9 | # Writing an issue against the offending project and linking to it here is good. 10 | # 11 | # Note: Changes to this file will automatically be used by other repos, referencing 12 | # this file from Github directly. It does not require packaging in edx-lint. 13 | 14 | # using LTS django version 15 | Django<5.0 16 | 17 | # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. 18 | # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html 19 | # See https://github.com/openedx/edx-platform/issues/35126 for more info 20 | elasticsearch<7.14.0 21 | 22 | # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected 23 | django-simple-history==3.0.0 24 | 25 | # Cause: https://github.com/openedx/edx-lint/issues/458 26 | # This can be unpinned once https://github.com/openedx/edx-lint/issues/459 has been resolved. 27 | pip<24.3 28 | 29 | # Cause: https://github.com/openedx/edx-lint/issues/475 30 | # This can be unpinned once https://github.com/openedx/edx-lint/issues/476 has been resolved. 31 | urllib3<2.3.0 32 | -------------------------------------------------------------------------------- /requirements/constraints.txt: -------------------------------------------------------------------------------- 1 | # Version constraints for pip-installation. 2 | # 3 | # This file doesn't install any packages. It specifies version constraints 4 | # that will be applied if a package is needed. 5 | # 6 | # When pinning something here, please provide an explanation of why. Ideally, 7 | # link to other information that will help people in the future to remove the 8 | # pin when possible. Writing an issue against the offending project and 9 | # linking to it here is good. 10 | 11 | -c common_constraints.txt -------------------------------------------------------------------------------- /requirements/pip.in: -------------------------------------------------------------------------------- 1 | -c constraints.txt 2 | # Core dependencies for installing other packages 3 | 4 | pip 5 | setuptools 6 | wheel 7 | 8 | -------------------------------------------------------------------------------- /requirements/pip.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.11 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | wheel==0.45.1 8 | # via -r requirements/pip.in 9 | 10 | # The following packages are considered to be unsafe in a requirements file: 11 | pip==24.2 12 | # via 13 | # -c requirements/common_constraints.txt 14 | # -r requirements/pip.in 15 | setuptools==80.8.0 16 | # via -r requirements/pip.in 17 | -------------------------------------------------------------------------------- /requirements/pip_tools.in: -------------------------------------------------------------------------------- 1 | # Dependencies to run compile tools 2 | -c constraints.txt 3 | 4 | pip-tools # Contains pip-compile, used to generate pip requirements files 5 | -------------------------------------------------------------------------------- /requirements/pip_tools.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.11 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | build==1.2.2.post1 8 | # via pip-tools 9 | click==8.2.1 10 | # via pip-tools 11 | packaging==25.0 12 | # via build 13 | pip-tools==7.4.1 14 | # via -r requirements/pip_tools.in 15 | pyproject-hooks==1.2.0 16 | # via 17 | # build 18 | # pip-tools 19 | wheel==0.45.1 20 | # via pip-tools 21 | 22 | # The following packages are considered to be unsafe in a requirements file: 23 | # pip 24 | # setuptools 25 | -------------------------------------------------------------------------------- /requirements/production.in: -------------------------------------------------------------------------------- 1 | # Production requirements for using this package 2 | 3 | -c constraints.txt 4 | 5 | -r base.txt 6 | -------------------------------------------------------------------------------- /requirements/production.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.11 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | -e git+https://github.com/openedx/codejail.git@4127fc4bd5775cc72aee8d7f0a70e31405e22439#egg=codejail 8 | # via -r requirements/base.txt 9 | certifi==2025.4.26 10 | # via 11 | # -r requirements/base.txt 12 | # requests 13 | charset-normalizer==3.4.2 14 | # via 15 | # -r requirements/base.txt 16 | # requests 17 | dogstatsd-python==0.5.6 18 | # via -r requirements/base.txt 19 | idna==3.10 20 | # via 21 | # -r requirements/base.txt 22 | # requests 23 | path==17.1.0 24 | # via 25 | # -r requirements/base.txt 26 | # path-py 27 | path-py==12.5.0 28 | # via -r requirements/base.txt 29 | requests==2.32.3 30 | # via -r requirements/base.txt 31 | six==1.17.0 32 | # via -r requirements/base.txt 33 | urllib3==2.2.3 34 | # via 35 | # -c requirements/common_constraints.txt 36 | # -r requirements/base.txt 37 | # requests 38 | -------------------------------------------------------------------------------- /requirements/test.in: -------------------------------------------------------------------------------- 1 | # Requirements for test runs 2 | 3 | -c constraints.txt 4 | 5 | -r production.txt 6 | 7 | mock 8 | pytest-cov 9 | -------------------------------------------------------------------------------- /requirements/test.txt: -------------------------------------------------------------------------------- 1 | # 2 | # This file is autogenerated by pip-compile with Python 3.11 3 | # by the following command: 4 | # 5 | # make upgrade 6 | # 7 | -e git+https://github.com/openedx/codejail.git@4127fc4bd5775cc72aee8d7f0a70e31405e22439#egg=codejail 8 | # via -r requirements/production.txt 9 | certifi==2025.4.26 10 | # via 11 | # -r requirements/production.txt 12 | # requests 13 | charset-normalizer==3.4.2 14 | # via 15 | # -r requirements/production.txt 16 | # requests 17 | coverage[toml]==7.8.1 18 | # via pytest-cov 19 | dogstatsd-python==0.5.6 20 | # via -r requirements/production.txt 21 | idna==3.10 22 | # via 23 | # -r requirements/production.txt 24 | # requests 25 | iniconfig==2.1.0 26 | # via pytest 27 | mock==5.2.0 28 | # via -r requirements/test.in 29 | packaging==25.0 30 | # via pytest 31 | path==17.1.0 32 | # via 33 | # -r requirements/production.txt 34 | # path-py 35 | path-py==12.5.0 36 | # via -r requirements/production.txt 37 | pluggy==1.6.0 38 | # via pytest 39 | pytest==8.3.5 40 | # via pytest-cov 41 | pytest-cov==6.1.1 42 | # via -r requirements/test.in 43 | requests==2.32.3 44 | # via -r requirements/production.txt 45 | six==1.17.0 46 | # via -r requirements/production.txt 47 | urllib3==2.2.3 48 | # via 49 | # -c requirements/common_constraints.txt 50 | # -r requirements/production.txt 51 | # requests 52 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name='xqueue_watcher', 6 | version='0.4', 7 | description='XQueue Pull Grader', 8 | packages=[ 9 | 'grader_support', 10 | 'xqueue_watcher', 11 | ], 12 | install_requires=open('requirements/production.txt', 13 | 'rt', encoding='utf-8').readlines(), 14 | ) 15 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx/xqueue-watcher/db4ed32a8f7a566df9674753ffeb02717cd2a296/tests/__init__.py -------------------------------------------------------------------------------- /tests/fixtures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx/xqueue-watcher/db4ed32a8f7a566df9674753ffeb02717cd2a296/tests/fixtures/__init__.py -------------------------------------------------------------------------------- /tests/fixtures/answer.py: -------------------------------------------------------------------------------- 1 | def foo(): 2 | return 'hi' 3 | -------------------------------------------------------------------------------- /tests/fixtures/config/conf.d/example.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /tests/fixtures/config/logging.json: -------------------------------------------------------------------------------- 1 | {"version": 1} -------------------------------------------------------------------------------- /tests/fixtures/fake_grader.py: -------------------------------------------------------------------------------- 1 | from grader_support import gradelib 2 | 3 | grader = gradelib.Grader() 4 | 5 | grader.add_test(gradelib.InvokeStudentFunctionTest('foo', [])) 6 | -------------------------------------------------------------------------------- /tests/test_grader.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | import json 4 | import sys 5 | from path import Path 6 | from queue import Queue 7 | 8 | from xqueue_watcher import grader 9 | 10 | MYDIR = Path(__file__).dirname() / 'fixtures' 11 | 12 | 13 | class MockGrader(grader.Grader): 14 | def grade(self, grader_path, grader_config, student_response): 15 | tests = [] 16 | errors = [] 17 | correct = 0 18 | score = 0 19 | if grader_path.endswith('/correct'): 20 | correct = 1 21 | score = 1 22 | tests.append(('short', 'long', True, 'expected', 'actual')) 23 | tests.append(('short', '', True, 'expected', 'actual')) 24 | elif grader_path.endswith('/incorrect'): 25 | tests.append(('short', 'long', False, 'expected', 'actual')) 26 | errors.append('THIS IS AN ERROR') 27 | errors.append('\x00\xc3\x83\xc3\xb8\x02') 28 | 29 | try: 30 | from codejail import jail_code 31 | except ImportError: 32 | tests.append(("codejail", "codejail not installed", True, "", "")) 33 | else: 34 | if jail_code.is_configured("python"): 35 | tests.append(("codejail", "codejail configured", True, "", "")) 36 | else: 37 | tests.append(("codejail", "codejail not configured", True, "", "")) 38 | 39 | results = { 40 | 'correct': correct, 41 | 'score': score, 42 | 'tests': tests, 43 | 'errors': errors, 44 | } 45 | return results 46 | 47 | 48 | class GraderTests(unittest.TestCase): 49 | def _make_payload(self, body, files=''): 50 | return { 51 | 'xqueue_body': json.dumps(body), 52 | 'xqueue_files': files 53 | } 54 | 55 | def test_bad_payload(self): 56 | g = MockGrader() 57 | 58 | self.assertRaises(KeyError, g.process_item, {}) 59 | self.assertRaises(ValueError, g.process_item, {'xqueue_body': '', 'xqueue_files': ''}) 60 | pl = self._make_payload({ 61 | 'student_response': 'blah', 62 | 'grader_payload': 'blah' 63 | }) 64 | self.assertRaises(ValueError, g.process_item, pl) 65 | 66 | def test_no_grader(self): 67 | g = grader.Grader() 68 | pl = self._make_payload({ 69 | 'student_response': 'blah', 70 | 'grader_payload': json.dumps({ 71 | 'grader': '/tmp/grader.py' 72 | }) 73 | }) 74 | self.assertRaises(NotImplementedError, g.process_item, pl) 75 | 76 | # grader that doesn't exist 77 | self.assertRaises(Exception, grader.Grader, gradepy='/asdfasdfdasf.py') 78 | 79 | def test_correct_response(self): 80 | g = MockGrader() 81 | pl = self._make_payload({ 82 | 'student_response': 'blah', 83 | 'grader_payload': json.dumps({ 84 | 'grader': 'correct' 85 | }) 86 | }) 87 | reply = g.process_item(pl) 88 | self.assertIn('result-correct', reply['msg']) 89 | self.assertEqual(reply['correct'], 1) 90 | self.assertEqual(reply['score'], 1) 91 | 92 | def test_incorrect_response(self): 93 | g = MockGrader() 94 | pl = self._make_payload({ 95 | 'student_response': 'blah', 96 | 'grader_payload': json.dumps({ 97 | 'grader': 'incorrect' 98 | }) 99 | }) 100 | reply = g.process_item(pl) 101 | self.assertIn('result-incorrect', reply['msg']) 102 | self.assertIn('THIS IS AN ERROR', reply['msg']) 103 | self.assertEqual(reply['correct'], 0) 104 | self.assertEqual(reply['score'], 0) 105 | 106 | def test_response_on_queue(self): 107 | g = MockGrader() 108 | pl = self._make_payload({ 109 | 'student_response': 'blah', 110 | 'grader_payload': json.dumps({ 111 | 'grader': 'correct' 112 | }) 113 | }) 114 | q = Queue() 115 | reply = g.process_item(pl, queue=q) 116 | popped = q.get() 117 | self.assertEqual(reply, popped) 118 | 119 | del pl['xqueue_body'] 120 | try: 121 | g.process_item(pl, queue=q) 122 | except Exception as e: 123 | popped = q.get() 124 | self.assertEqual(e, popped) 125 | 126 | def test_subprocess(self): 127 | g = MockGrader() 128 | pl = self._make_payload({ 129 | 'student_response': 'blah', 130 | 'grader_payload': json.dumps({ 131 | 'grader': 'correct' 132 | }) 133 | }) 134 | reply = g(pl) 135 | self.assertEqual(reply['correct'], 1) 136 | 137 | del pl['xqueue_body'] 138 | 139 | self.assertRaises(KeyError, g, pl) 140 | 141 | def test_no_fork(self): 142 | g = MockGrader(fork_per_item=False) 143 | pl = self._make_payload({ 144 | 'student_response': 'blah', 145 | 'grader_payload': json.dumps({ 146 | 'grader': 'correct' 147 | }) 148 | }) 149 | reply = g(pl) 150 | self.assertEqual(reply['correct'], 1) 151 | -------------------------------------------------------------------------------- /tests/test_jailed_grader.py: -------------------------------------------------------------------------------- 1 | import getpass 2 | import os 3 | import sys 4 | import textwrap 5 | import unittest 6 | from path import Path 7 | 8 | from xqueue_watcher.jailedgrader import JailedGrader 9 | from codejail.jail_code import configure 10 | 11 | 12 | class JailedGraderTests(unittest.TestCase): 13 | def setUp(self): 14 | configure("python", sys.executable, user=getpass.getuser()) 15 | py3paths = [os.environ.get('XQUEUEWATCHER_PYTHON3_BIN'), '/usr/bin/python3', '/usr/local/bin/python3'] 16 | for py3path in (p3p for p3p in py3paths if p3p): 17 | if os.path.exists(py3path): 18 | configure( 19 | "python3", 20 | py3path, 21 | user=getpass.getuser(), 22 | ) 23 | break 24 | self.grader_root = Path(__file__).dirname() / 'fixtures' 25 | self.g = JailedGrader(grader_root=self.grader_root) 26 | self.g3 = JailedGrader(grader_root=self.grader_root, codejail_python='python3') 27 | 28 | def test_correct(self): 29 | code = textwrap.dedent(''' 30 | def foo(): 31 | return "hi" 32 | ''') 33 | response = self.g.grade(self.grader_root / 'fake_grader.py', {}, code) 34 | self.assertEqual(response['score'], 1) 35 | 36 | def test_incorrect(self): 37 | code = textwrap.dedent(''' 38 | def zoo(): 39 | return "hi" 40 | ''') 41 | response = self.g.grade(self.grader_root / 'fake_grader.py', {}, code) 42 | self.assertEqual(response['score'], 0) 43 | 44 | response = self.g.grade(self.grader_root / 'fake_grader.py', {}, '') 45 | self.assertEqual(response['score'], 0) 46 | 47 | response = self.g.grade(self.grader_root / 'fake_grader.py', {}, 'asdofhpsdfuh') 48 | self.assertEqual(response['score'], 0) 49 | -------------------------------------------------------------------------------- /tests/test_manager.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from path import Path 3 | import json 4 | from unittest.mock import Mock 5 | import time 6 | import sys 7 | 8 | import logging 9 | from xqueue_watcher import manager 10 | from tests.test_xqueue_client import MockXQueueServer 11 | 12 | from io import StringIO 13 | 14 | try: 15 | import codejail 16 | HAS_CODEJAIL = True 17 | except ImportError: 18 | HAS_CODEJAIL = False 19 | 20 | 21 | class ManagerTests(unittest.TestCase): 22 | def setUp(self): 23 | self.m = manager.Manager() 24 | self.config = { 25 | 'test1': { 26 | 'SERVER': 'http://test1', 27 | 'AUTH': ('test', 'test'), 28 | 'HANDLERS': [ 29 | { 30 | 'HANDLER': 'tests.test_grader.MockGrader', 31 | } 32 | ] 33 | }, 34 | 'test2': { 35 | 'AUTH': ('test', 'test'), 36 | 'CONNECTIONS': 2, 37 | 'SERVER': 'http://test2', 38 | 'HANDLERS': [ 39 | { 40 | 'HANDLER': 'urllib.urlencode' 41 | } 42 | ] 43 | } 44 | } 45 | 46 | def tearDown(self): 47 | try: 48 | self.m.shutdown() 49 | except SystemExit: 50 | pass 51 | 52 | def test_configuration(self): 53 | self.m.configure(self.config) 54 | self.assertEqual(len(self.m.clients), 3) 55 | for c in self.m.clients: 56 | if c.queue_name == 'test2': 57 | self.assertEqual(c.xqueue_server, 'http://test2') 58 | 59 | @unittest.skipUnless(HAS_CODEJAIL, "Codejail not installed") 60 | def test_codejail_config(self): 61 | config = { 62 | "name": "python", 63 | "bin_path": "/usr/bin/python", 64 | "user": "nobody", 65 | "limits": { 66 | "CPU": 2, 67 | "VMEM": 1024 68 | } 69 | } 70 | codejail_return = self.m.enable_codejail(config) 71 | self.assertEqual(codejail_return, config["name"]) 72 | self.assertTrue(codejail.jail_code.is_configured("python")) 73 | self.m.enable_codejail({ 74 | "name": "other-python", 75 | "bin_path": "/usr/local/bin/python" 76 | }) 77 | self.assertTrue(codejail.jail_code.is_configured("other-python")) 78 | 79 | # now we'll see if the codejail config is inherited in the handler subprocess 80 | handler_config = self.config['test1'].copy() 81 | client = self.m.client_from_config("test", handler_config) 82 | client.session = MockXQueueServer() 83 | client._handle_submission(json.dumps({ 84 | "xqueue_header": "", 85 | "xqueue_files": [], 86 | "xqueue_body": json.dumps({ 87 | 'student_response': 'blah', 88 | 'grader_payload': json.dumps({ 89 | 'grader': '/tmp/grader.py' 90 | }) 91 | }) 92 | })) 93 | last_req = client.session._requests[-1] 94 | self.assertIn('codejail configured', last_req.kwargs['data']['xqueue_body']) 95 | 96 | def test_start(self): 97 | self.m.configure(self.config) 98 | sess = MockXQueueServer() 99 | sess._json = {'return_code': 0, 'msg': 'logged in'} 100 | for c in self.m.clients: 101 | c.session = sess 102 | 103 | self.m.start() 104 | for c in self.m.clients: 105 | self.assertTrue(c.is_alive()) 106 | 107 | def test_shutdown(self): 108 | self.m.configure(self.config) 109 | sess = MockXQueueServer() 110 | sess._json = {'return_code': 0, 'msg': 'logged in'} 111 | for c in self.m.clients: 112 | c.session = sess 113 | self.m.start() 114 | self.assertRaises(SystemExit, self.m.shutdown) 115 | 116 | def test_wait(self): 117 | # no-op 118 | self.m.wait() 119 | 120 | self.m.configure(self.config) 121 | 122 | def slow_reply(url, response, session): 123 | if url.endswith('get_submission/'): 124 | response.json.return_value = { 125 | 'return_code': 0, 126 | 'success': 1, 127 | 'content': json.dumps({ 128 | 'xqueue_header': {'hello': 1}, 129 | 'xqueue_body': { 130 | 'blah': json.dumps({}), 131 | } 132 | }) 133 | } 134 | if url.endswith('put_result/'): 135 | time.sleep(2) 136 | 137 | import threading 138 | def stopper(client): 139 | time.sleep(.4) 140 | client.running = False 141 | 142 | for c in self.m.clients: 143 | c.session = MockXQueueServer() 144 | c._json = {'return_code': 0, 'msg': 'logged in'} 145 | c.session._url_checker = slow_reply 146 | c.session._json = {'return_code': 0} 147 | 148 | self.m.poll_time = 1 149 | self.m.start() 150 | threading.Thread(target=stopper, args=(self.m.clients[0],)).start() 151 | 152 | self.assertRaises(SystemExit, self.m.wait) 153 | 154 | def test_main_with_errors(self): 155 | stderr = sys.stderr 156 | sys.stderr = StringIO() 157 | self.assertRaises(SystemExit, manager.main, []) 158 | sys.stderr.seek(0) 159 | err_msg = sys.stderr.read() 160 | self.assertIn('usage: xqueue_watcher [-h] -d CONFIG_ROOT', err_msg) 161 | self.assertIn('-d/--config_root', err_msg) 162 | self.assertIn('required', err_msg) 163 | sys.stderr = stderr 164 | 165 | mydir = Path(__file__).dirname() 166 | args = ['-d', mydir / "fixtures/config"] 167 | self.assertEqual(manager.main(args), 0) 168 | -------------------------------------------------------------------------------- /tests/test_xqueue_client.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | import json 4 | import collections 5 | import requests 6 | import requests.exceptions 7 | 8 | from xqueue_watcher import client 9 | 10 | Request = collections.namedtuple('Request', ('method', 'url', 'kwargs', 'response')) 11 | 12 | 13 | class MockXQueueServer(mock.Mock): 14 | def __init__(self): 15 | mock.Mock.__init__(self) 16 | self.status_code = 200 17 | self._json = None 18 | self._requests = [] 19 | self._loginok = True 20 | self._fail = False 21 | self._url_checker = None 22 | self._open = True 23 | 24 | def close(self): 25 | self._open = False 26 | 27 | def request(self, method, url, **kwargs): 28 | response = mock.Mock() 29 | response.json = mock.MagicMock() 30 | if self._json: 31 | if isinstance(self._json, Exception): 32 | response.json.side_effect = self._json 33 | else: 34 | response.json.return_value = self._json 35 | response.status_code = self.status_code 36 | 37 | if self._url_checker: 38 | self._url_checker(url, response, self) 39 | 40 | self._requests.append(Request(method, url, kwargs, response)) 41 | if self._fail: 42 | raise self._fail 43 | return response 44 | 45 | 46 | class ClientTests(unittest.TestCase): 47 | def setUp(self): 48 | self.client = client.XQueueClient('test', xqueue_server='TEST') 49 | self.session = MockXQueueServer() 50 | self.client.session = self.session 51 | self.qitem = None 52 | self.excepted = False 53 | 54 | self.sample_item = { 55 | 'return_code': 0, 56 | 'success': 1, 57 | 'content': json.dumps({ 58 | 'xqueue_header': {'hello': 1}, 59 | 'xqueue_body': { 60 | 'blah': 'blah' 61 | } 62 | }) 63 | } 64 | self.session._json = self.sample_item 65 | 66 | def _simple_handler(self, content): 67 | self.qitem = content 68 | 69 | def test_repr(self): 70 | self.assertEqual(repr(self.client), 'XQueueClient(%s)' % self.client.queue_name) 71 | 72 | def test_process_one(self): 73 | self.client.add_handler(self._simple_handler) 74 | reply = self.client.process_one() 75 | self.assertTrue(reply) 76 | self.assertTrue(self.qitem is not None) 77 | self.assertEqual(self.qitem, json.loads(self.sample_item['content'])) 78 | 79 | # try with different return_code 80 | del self.sample_item['return_code'] 81 | reply = self.client.process_one() 82 | self.assertTrue(reply) 83 | self.assertTrue(self.qitem is not None) 84 | self.assertEqual(self.qitem, json.loads(self.sample_item['content'])) 85 | 86 | # try with wrong return code 87 | self.sample_item['success'] = 'bad' 88 | reply = self.client.process_one() 89 | self.assertFalse(reply) 90 | 91 | # try with no return code 92 | del self.sample_item['success'] 93 | reply = self.client.process_one() 94 | self.assertFalse(reply) 95 | 96 | def test_add_remove(self): 97 | def handler(content): 98 | self.qitem = content 99 | 100 | self.client.add_handler(handler) 101 | reply = self.client.process_one() 102 | self.assertTrue(self.qitem is not None) 103 | self.qitem = None 104 | 105 | self.client.remove_handler(handler) 106 | reply = self.client.process_one() 107 | self.assertTrue(self.qitem is None) 108 | 109 | def test_handler_exception(self): 110 | def raises(content): 111 | self.excepted = True 112 | self.qitem = content 113 | raise Exception('test') 114 | 115 | self.client.add_handler(raises) 116 | reply = self.client.process_one() 117 | self.assertTrue(reply) 118 | self.assertTrue(self.excepted) 119 | self.assertTrue(self.qitem is not None) 120 | 121 | def test_bad_json(self): 122 | self.client.add_handler(self._simple_handler) 123 | self.session._json = ValueError() 124 | reply = self.client.process_one() 125 | self.assertFalse(reply) 126 | 127 | def test_bad_connection(self): 128 | self.client.add_handler(self._simple_handler) 129 | self.session.status_code = 500 130 | reply = self.client.process_one() 131 | self.assertFalse(reply) 132 | 133 | # connection exception 134 | self.session._fail = requests.exceptions.ConnectionError() 135 | reply = self.client.process_one() 136 | self.assertFalse(reply) 137 | 138 | # handle timeout 139 | self.session._fail = requests.exceptions.Timeout() 140 | reply = self.client.process_one() 141 | self.assertTrue(reply) 142 | 143 | def test_redirect_to_login(self): 144 | self.client.add_handler(self._simple_handler) 145 | self.session.status_code = 302 146 | 147 | def login(url, response, session): 148 | if url.endswith('xqueue/login/'): 149 | response.status_code = 200 150 | response.json.return_value = {'return_code': 0, 'msg': 'logged in'} 151 | session.status_code = 200 152 | self.session._url_checker = login 153 | 154 | reply = self.client.process_one() 155 | req = self.session._requests[1] 156 | self.assertTrue(req.url, 'TEST/xqueue/login/') 157 | self.assertTrue(reply) 158 | 159 | def test_bad_login(self): 160 | self.client.add_handler(self._simple_handler) 161 | self.session.status_code = 302 162 | 163 | def login(url, response, session): 164 | if url.endswith('xqueue/login/'): 165 | response.status_code = 200 166 | response.json.return_value = {'return_code': 1, 'msg': 'bad login'} 167 | session.status_code = 200 168 | 169 | self.session._url_checker = login 170 | 171 | reply = self.client.process_one() 172 | req = self.session._requests[1] 173 | self.assertTrue(req.url, 'TEST/xqueue/login/') 174 | self.assertFalse(reply) 175 | 176 | def test_post_back(self): 177 | def handler(content): 178 | return {'result': True} 179 | 180 | self.client.add_handler(handler) 181 | result = self.client.process_one() 182 | self.assertTrue(result) 183 | last_request = self.session._requests[-1] 184 | self.assertTrue(last_request.url.endswith('put_result/')) 185 | posted = last_request.kwargs['data'] 186 | self.assertEqual(posted['xqueue_body'], json.dumps({'result': True})) 187 | 188 | # test failure case 189 | def postfailure(url, response, session): 190 | if url.endswith('put_result/'): 191 | response.status_code = 500 192 | self.session._url_checker = postfailure 193 | result = self.client.process_one() 194 | self.assertFalse(result) 195 | last_request = self.session._requests[-1] 196 | self.assertTrue(last_request.url.endswith('put_result/')) 197 | 198 | def test_run(self): 199 | def handler(content): 200 | return {'result': True} 201 | 202 | def urlchecker(url, response, session): 203 | if url.endswith('/login/'): 204 | response.status_code = 200 205 | response.json.return_value = {'return_code': 0, 'msg': 'logged in'} 206 | self.session.status_code = 200 207 | elif url.endswith('get_submission/') and len(session._requests) > 3: 208 | self.client.shutdown() 209 | response.status_code = 500 210 | 211 | self.session._url_checker = urlchecker 212 | self.client.add_handler(handler) 213 | 214 | self.client.run() 215 | self.assertFalse(self.client.running) 216 | self.assertFalse(self.session._open) 217 | 218 | # test failed login 219 | def urlchecker(url, response, session): 220 | if url.endswith('/login/'): 221 | response.status_code = 500 222 | 223 | self.session._url_checker = urlchecker 224 | self.client.running = False 225 | self.assertTrue(self.client.run()) 226 | -------------------------------------------------------------------------------- /xqueue_watcher/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/openedx/xqueue-watcher/db4ed32a8f7a566df9674753ffeb02717cd2a296/xqueue_watcher/__init__.py -------------------------------------------------------------------------------- /xqueue_watcher/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from .manager import main 3 | 4 | sys.exit(main()) 5 | -------------------------------------------------------------------------------- /xqueue_watcher/client.py: -------------------------------------------------------------------------------- 1 | import time 2 | import json 3 | import logging 4 | import requests 5 | from requests.auth import HTTPBasicAuth 6 | import threading 7 | import multiprocessing 8 | from .settings import MANAGER_CONFIG_DEFAULTS 9 | 10 | log = logging.getLogger(__name__) 11 | 12 | 13 | class XQueueClient: 14 | def __init__(self, 15 | queue_name, 16 | xqueue_server='http://localhost:18040', 17 | xqueue_auth=('user', 'pass'), 18 | http_basic_auth=MANAGER_CONFIG_DEFAULTS['HTTP_BASIC_AUTH'], 19 | requests_timeout=MANAGER_CONFIG_DEFAULTS['REQUESTS_TIMEOUT'], 20 | poll_interval=MANAGER_CONFIG_DEFAULTS['POLL_INTERVAL'], 21 | login_poll_interval=MANAGER_CONFIG_DEFAULTS['LOGIN_POLL_INTERVAL'], 22 | follow_client_redirects=MANAGER_CONFIG_DEFAULTS['FOLLOW_CLIENT_REDIRECTS']): 23 | super().__init__() 24 | self.session = requests.session() 25 | self.xqueue_server = xqueue_server 26 | self.queue_name = queue_name 27 | self.handlers = [] 28 | self.daemon = True 29 | self.username, self.password = xqueue_auth 30 | self.requests_timeout = requests_timeout 31 | self.poll_interval = poll_interval 32 | self.login_poll_interval = login_poll_interval 33 | self.follow_client_redirects = follow_client_redirects 34 | 35 | if http_basic_auth is not None: 36 | self.http_basic_auth = HTTPBasicAuth(*http_basic_auth) 37 | else: 38 | self.http_basic_auth = None 39 | 40 | self.running = True 41 | self.processing = False 42 | 43 | def __repr__(self): 44 | return f'{self.__class__.__name__}({self.queue_name})' 45 | 46 | def _parse_response(self, response, is_reply=True): 47 | if response.status_code not in [200]: 48 | error_message = "Server %s returned status_code=%d" % (response.url, response.status_code) 49 | log.error(error_message) 50 | return False, error_message 51 | 52 | try: 53 | xreply = response.json() 54 | except ValueError: 55 | error_message = "Could not parse xreply." 56 | log.error(error_message) 57 | return False, error_message 58 | 59 | if 'return_code' in xreply: 60 | return_code = xreply['return_code'] == 0 61 | content = xreply['content'] 62 | elif 'success' in xreply: 63 | return_code = xreply['success'] 64 | content = xreply 65 | else: 66 | return False, "Cannot find a valid success or return code." 67 | 68 | if return_code not in [True, False]: 69 | return False, 'Invalid return code.' 70 | 71 | return return_code, content 72 | 73 | def _request(self, method, uri, **kwargs): 74 | url = self.xqueue_server + uri 75 | r = None 76 | while not r: 77 | try: 78 | r = self.session.request( 79 | method, 80 | url, 81 | auth=self.http_basic_auth, 82 | timeout=self.requests_timeout, 83 | allow_redirects=self.follow_client_redirects, 84 | **kwargs 85 | ) 86 | except requests.exceptions.ConnectionError as e: 87 | log.error('Could not connect to server at %s in timeout=%r', url, self.requests_timeout) 88 | return (False, e) 89 | if r.status_code == 200: 90 | return self._parse_response(r) 91 | # Django can issue both a 302 to the login page and a 92 | # 301 if the original URL did not have a trailing / and 93 | # APPEND_SLASH is true in XQueue deployment, which is the default. 94 | elif r.status_code in (301, 302): 95 | if self._login(): 96 | r = None 97 | else: 98 | return (False, "Could not log in") 99 | else: 100 | message = "Received un expected response status code, {}, calling {}.".format( 101 | r.status_code,url) 102 | log.error(message) 103 | return (False, message) 104 | 105 | def _login(self): 106 | if self.username is None: 107 | return True 108 | url = self.xqueue_server + '/xqueue/login/' 109 | log.debug(f"Trying to login to {url} with user: {self.username} and pass {self.password}") 110 | response = self.session.request('post', url, auth=self.http_basic_auth, data={ 111 | 'username': self.username, 112 | 'password': self.password, 113 | }) 114 | if response.status_code != 200: 115 | log.error('Log in error %s %s', response.status_code, response.content) 116 | return False 117 | msg = response.json() 118 | log.debug("login response from %r: %r", url, msg) 119 | return msg['return_code'] == 0 120 | 121 | def shutdown(self): 122 | """ 123 | Close connection and shutdown 124 | """ 125 | self.running = False 126 | self.session.close() 127 | 128 | def add_handler(self, handler): 129 | """ 130 | Add handler function to be called for every item in the queue 131 | """ 132 | self.handlers.append(handler) 133 | 134 | def remove_handler(self, handler): 135 | """ 136 | Remove handler function 137 | """ 138 | self.handlers.remove(handler) 139 | 140 | def _handle_submission(self, content): 141 | content = json.loads(content) 142 | success = [] 143 | for handler in self.handlers: 144 | result = handler(content) 145 | if result: 146 | reply = {'xqueue_body': json.dumps(result), 147 | 'xqueue_header': content['xqueue_header']} 148 | status, message = self._request('post', '/xqueue/put_result/', data=reply, verify=False) 149 | if not status: 150 | log.error('Failure for %r -> %r', reply, message) 151 | success.append(status) 152 | return all(success) 153 | 154 | def process_one(self): 155 | try: 156 | self.processing = False 157 | get_params = {'queue_name': self.queue_name} 158 | success, content = self._request('get', '/xqueue/get_submission/', params=get_params) 159 | if success: 160 | self.processing = True 161 | success = self._handle_submission(content) 162 | return success 163 | except requests.exceptions.Timeout: 164 | return True 165 | except Exception as e: 166 | log.exception(e) 167 | return True 168 | 169 | def run(self): 170 | """ 171 | Run forever, processing items from the queue 172 | """ 173 | if not self._login(): 174 | log.error("Could not log in to Xqueue %s. Retrying every 5 seconds..." % self.queue_name) 175 | num_tries = 1 176 | while self.running: 177 | num_tries += 1 178 | time.sleep(self.login_poll_interval) 179 | if not self._login(): 180 | log.error("Still could not log in to %s (%s:%s) tries: %d", 181 | self.queue_name, 182 | self.username, 183 | self.password, 184 | num_tries) 185 | else: 186 | break 187 | while self.running: 188 | if not self.process_one(): 189 | time.sleep(self.poll_interval) 190 | return True 191 | 192 | 193 | class XQueueClientThread(XQueueClient, threading.Thread): 194 | pass 195 | 196 | 197 | class XQueueClientProcess(XQueueClient, multiprocessing.Process): 198 | pass 199 | -------------------------------------------------------------------------------- /xqueue_watcher/grader.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implementation of a grader compatible with XServer 3 | """ 4 | import html 5 | import os 6 | import time 7 | import json 8 | from path import Path 9 | import logging 10 | import multiprocessing 11 | from statsd import statsd 12 | 13 | 14 | def format_errors(errors): 15 | esc = html.escape 16 | error_string = '' 17 | error_list = [esc(e) for e in errors or []] 18 | if error_list: 19 | items = '\n'.join([f'
  • {e}
    • \n' for e in error_list]) 20 | error_string = f'
      \n{items}
    \n' 21 | error_string = f'
    {error_string}
    ' 22 | return error_string 23 | 24 | 25 | def to_dict(result): 26 | # long description may or may not be provided. If not, don't display it. 27 | # TODO: replace with mako template 28 | esc = html.escape 29 | if result[1]: 30 | long_desc = '

    {}

    '.format(esc(result[1])) 31 | else: 32 | long_desc = '' 33 | return {'short-description': esc(result[0]), 34 | 'long-description': long_desc, 35 | 'correct': result[2], # Boolean; don't escape. 36 | 'expected-output': esc(result[3]), 37 | 'actual-output': esc(result[4]) 38 | } 39 | 40 | 41 | class Grader: 42 | results_template = """ 43 |
    44 |
    Test results
    45 |
    46 |
    47 | {status} 48 |
    49 |
    50 | {errors} 51 | {results} 52 |
    53 |
    54 |
    55 | """ 56 | 57 | results_correct_template = """ 58 |
    59 |

    {short-description}

    60 | 
    {long-description}
    61 |
    62 |
    Output:
    63 |
    64 | 
    {actual-output}
    65 |
    66 |
    67 |
    68 | """ 69 | 70 | results_incorrect_template = """ 71 |
    72 |

    {short-description}

    73 | 
    {long-description}
    74 |
    75 |
    Your output:
    76 | 
    {actual-output}
    77 |
    Correct output:
    78 | 
    {expected-output}
    79 |
    80 |
    81 | """ 82 | 83 | def __init__(self, grader_root='/tmp/', fork_per_item=True, logger_name=__name__): 84 | """ 85 | grader_root = root path to graders 86 | fork_per_item = fork a process for every request 87 | logger_name = name of logger 88 | """ 89 | self.log = logging.getLogger(logger_name) 90 | self.grader_root = Path(grader_root) 91 | 92 | self.fork_per_item = fork_per_item 93 | 94 | def __call__(self, content): 95 | if self.fork_per_item: 96 | q = multiprocessing.Queue() 97 | proc = multiprocessing.Process(target=self.process_item, args=(content, q)) 98 | proc.start() 99 | proc.join() 100 | reply = q.get_nowait() 101 | if isinstance(reply, Exception): 102 | raise reply 103 | else: 104 | return reply 105 | else: 106 | return self.process_item(content) 107 | 108 | def grade(self, grader_path, grader_config, student_response): 109 | raise NotImplementedError("no grader defined") 110 | 111 | def process_item(self, content, queue=None): 112 | try: 113 | statsd.increment('xqueuewatcher.process-item') 114 | body = content['xqueue_body'] 115 | files = content['xqueue_files'] 116 | 117 | # Delivery from the lms 118 | body = json.loads(body) 119 | student_response = body['student_response'] 120 | payload = body['grader_payload'] 121 | try: 122 | grader_config = json.loads(payload) 123 | except ValueError as err: 124 | # If parsing json fails, erroring is fine--something is wrong in the content. 125 | # However, for debugging, still want to see what the problem is 126 | statsd.increment('xqueuewatcher.grader_payload_error') 127 | 128 | self.log.debug(f"error parsing: '{payload}' -- {err}") 129 | raise 130 | 131 | self.log.debug(f"Processing submission, grader payload: {payload}") 132 | relative_grader_path = grader_config['grader'] 133 | grader_path = os.path.abspath(self.grader_root / relative_grader_path) 134 | start = time.time() 135 | results = self.grade(grader_path, grader_config, student_response) 136 | 137 | statsd.histogram('xqueuewatcher.grading-time', time.time() - start) 138 | 139 | # Make valid JSON message 140 | reply = {'correct': results['correct'], 141 | 'score': results['score'], 142 | 'msg': self.render_results(results)} 143 | 144 | statsd.increment('xqueuewatcher.replies (non-exception)') 145 | except Exception as e: 146 | self.log.exception("process_item") 147 | if queue: 148 | queue.put(e) 149 | else: 150 | raise 151 | else: 152 | if queue: 153 | queue.put(reply) 154 | return reply 155 | 156 | def render_results(self, results): 157 | output = [] 158 | test_results = [to_dict(r) for r in results['tests']] 159 | for result in test_results: 160 | if result['correct']: 161 | template = self.results_correct_template 162 | else: 163 | template = self.results_incorrect_template 164 | output += template.format(**result) 165 | 166 | errors = format_errors(results['errors']) 167 | 168 | status = 'INCORRECT' 169 | if errors: 170 | status = 'ERROR' 171 | elif results['correct']: 172 | status = 'CORRECT' 173 | 174 | return self.results_template.format(status=status, 175 | errors=errors, 176 | results=''.join(output)) 177 | -------------------------------------------------------------------------------- /xqueue_watcher/jailedgrader.py: -------------------------------------------------------------------------------- 1 | """ 2 | An implementation of a grader that uses codejail to sandbox submission execution. 3 | """ 4 | import codecs 5 | import os 6 | import sys 7 | import importlib 8 | import json 9 | import random 10 | import gettext 11 | from path import Path 12 | import six 13 | 14 | import codejail 15 | 16 | from grader_support.gradelib import EndTest 17 | from grader_support.graderutil import LANGUAGE 18 | import grader_support 19 | 20 | from .grader import Grader 21 | 22 | TIMEOUT = 1 23 | 24 | def path_to_six(): 25 | """ 26 | Return the full path to six.py 27 | """ 28 | if any(six.__file__.endswith(suffix) for suffix in ('.pyc', '.pyo')): 29 | # __file__ points to the compiled bytecode in python 2 30 | return Path(six.__file__[:-1]) 31 | else: 32 | # __file__ points to the .py file in python 3 33 | return Path(six.__file__) 34 | 35 | 36 | SUPPORT_FILES = [ 37 | Path(grader_support.__file__).dirname(), 38 | path_to_six(), 39 | ] 40 | 41 | 42 | def truncate(out): 43 | """ 44 | Truncate test output that's too long. This is per-test. 45 | """ 46 | TOO_LONG = 5000 # 5K bytes seems like enough for a single test. 47 | if len(out) > TOO_LONG: 48 | out = out[:TOO_LONG] + "...OUTPUT TRUNCATED" 49 | 50 | return out 51 | 52 | 53 | def prepend_coding(code): 54 | """ 55 | Add a coding line--makes submissions with inline unicode not 56 | explode (as long as they're utf8, I guess) 57 | """ 58 | return '# coding: utf8\n' + code 59 | 60 | 61 | class JailedGrader(Grader): 62 | """ 63 | A grader implementation that uses codejail. 64 | Instantiate it with grader_root="path/to/graders" 65 | and optionally codejail_python="python name" (the name that you used to configure codejail) 66 | """ 67 | def __init__(self, *args, **kwargs): 68 | self.codejail_python = kwargs.pop("codejail_python", "python") 69 | super().__init__(*args, **kwargs) 70 | self.locale_dir = self.grader_root / "conf" / "locale" 71 | self.fork_per_item = False # it's probably safe not to fork 72 | # EDUCATOR-3368: OpenBLAS library is allowed to allocate 1 thread 73 | os.environ["OPENBLAS_NUM_THREADS"] = "1" 74 | 75 | def _enable_i18n(self, language): 76 | trans = gettext.translation('graders', localedir=self.locale_dir, fallback=True, languages=[language]) 77 | trans.install(names=None) 78 | 79 | def _run(self, grader_path, thecode, seed): 80 | files = SUPPORT_FILES + [grader_path] 81 | if self.locale_dir.exists(): 82 | files.append(self.locale_dir) 83 | extra_files = [('submission.py', thecode.encode('utf-8'))] 84 | argv = ["-m", "grader_support.run", Path(grader_path).basename(), 'submission.py', seed] 85 | r = codejail.jail_code.jail_code(self.codejail_python, files=files, extra_files=extra_files, argv=argv) 86 | return r 87 | 88 | def grade(self, grader_path, grader_config, submission): 89 | if type(submission) != str: 90 | self.log.warning("Submission is NOT unicode") 91 | 92 | results = { 93 | 'errors': [], 94 | 'tests': [], 95 | 'correct': False, 96 | 'score': 0, 97 | } 98 | 99 | # There are some cases where the course team would like to accept a 100 | # student submission but not process the student code. Some examples are 101 | # cases where the problem would require dependencies that are difficult 102 | # or impractical to install in a sandbox or if the complexity of the 103 | # solution would cause the runtime of the student code to exceed what is 104 | # possible in the sandbox. 105 | 106 | # skip_grader is a flag in the grader config which is a boolean. If it 107 | # is set to true on a problem then it will always show that the 108 | # submission is correct and give the student a full score for the 109 | # problem. 110 | if grader_config.get('skip_grader', False): 111 | results['correct'] = True 112 | results['score'] = 1 113 | self.log.debug('Skipping the grader.') 114 | return results 115 | 116 | self._enable_i18n(grader_config.get("lang", LANGUAGE)) 117 | 118 | answer_path = Path(grader_path).dirname() / 'answer.py' 119 | with open(answer_path, 'rb') as f: 120 | answer = f.read().decode('utf-8') 121 | 122 | # Import the grader, straight from the original file. (It probably isn't in 123 | # sys.path, and we may be in a long running gunicorn process, so we don't 124 | # want to add stuff to sys.path either.) 125 | sf_loader = importlib.machinery.SourceFileLoader("grader_module", str(grader_path)) 126 | grader_module = sf_loader.load_module() 127 | grader = grader_module.grader 128 | 129 | # Preprocess for grader-specified errors 130 | errors = grader.input_errors(submission) 131 | if errors != []: 132 | results['errors'].extend(errors) 133 | # Don't run tests if there were errors 134 | return results 135 | 136 | # Add a unicode encoding declaration. 137 | processed_answer = prepend_coding(grader.preprocess(answer)) 138 | processed_submission = prepend_coding(grader.preprocess(submission)) 139 | 140 | # Same seed for both runs 141 | seed = str(random.randint(0, 20000)) 142 | 143 | # Run the official answer, to get the expected output. 144 | expected_ok = False 145 | expected_exc = None 146 | try: 147 | # If we want a factor of two speedup for now: trust the staff solution to 148 | # avoid hitting the sandbox. (change run to run_trusted) 149 | expected_outputs = None # in case run_trusted raises an exception. 150 | expected_outputs = self._run(grader_path, processed_answer, seed).stdout 151 | if expected_outputs: 152 | expected = json.loads(expected_outputs.decode('utf-8')) 153 | expected_ok = True 154 | except Exception: 155 | expected_exc = sys.exc_info() 156 | else: 157 | # We just ran the official answer, nothing should have gone wrong, so check 158 | # everything, and note it as bad if anything is wrong. 159 | if expected_ok: 160 | if expected['exceptions'] \ 161 | or expected['grader']['status'] != 'ok' \ 162 | or expected['submission']['status'] != 'ok': 163 | expected_ok = False 164 | 165 | if not expected_ok: 166 | # We couldn't run the official answer properly, bail out, but don't show 167 | # details to the student, since none of it is their code. 168 | results['errors'].append(_('There was a problem running the staff solution (Staff debug: L364)')) 169 | self.log.error("Couldn't run staff solution. grader = %s, output: %r", 170 | grader_path, expected_outputs, exc_info=expected_exc) 171 | return results 172 | 173 | # The expected code ran fine, go ahead and run the student submission. 174 | actual_ok = False 175 | actual_exc = None 176 | try: 177 | # Do NOT trust the student solution (in production). 178 | actual_outputs = None # in case run raises an exception. 179 | actual_outputs = self._run(grader_path, processed_submission, seed).stdout 180 | if actual_outputs: 181 | actual = json.loads(actual_outputs.decode('utf-8')) 182 | actual_ok = True 183 | else: 184 | results['errors'].append(_("There was a problem running your solution (Staff debug: L379).")) 185 | except Exception: 186 | actual_exc = sys.exc_info() 187 | else: 188 | if actual_ok and actual['grader']['status'] == 'ok': 189 | if actual['submission']['status'] != 'ok': 190 | # The grader ran OK, but the student code didn't, so show the student 191 | # details of what went wrong. There is probably an exception to show. 192 | shown_error = actual['submission']['exception'] or _('There was an error thrown while running your solution.') 193 | results['errors'].append(shown_error) 194 | else: 195 | # The grader didn't run well, we are going to bail. 196 | actual_ok = False 197 | 198 | # If something went wrong, then don't continue 199 | if not actual_ok: 200 | results['errors'].append(_("We couldn't run your solution (Staff debug: L397).")) 201 | self.log.error("Couldn't run student solution. grader = %s, output: %r", 202 | grader_path, actual_outputs, exc_info=actual_exc) 203 | return results 204 | 205 | # Compare actual and expected through the grader tests, but only if we haven't 206 | # already found a problem. 207 | corrects = [] 208 | if not results['errors']: 209 | expected_results = expected['results'] 210 | actual_results = actual['results'] 211 | if len(expected_results) != len(actual_results): 212 | results['errors'].append(_('Something went wrong: different numbers of ' 213 | 'tests ran for your code and for our reference code.')) 214 | return results 215 | 216 | for test, exp, act in zip(grader.tests(), expected_results, actual_results): 217 | exp_short_desc, exp_long_desc, exp_output = exp 218 | act_short_desc, act_long_desc, act_output = act 219 | if exp_short_desc != act_short_desc: 220 | results['errors'].append(_("Something went wrong: tests don't match up.")) 221 | # TODO: don't give up so easily? 222 | return results 223 | # Truncate here--we don't want to send long output back, and also don't want to 224 | # confuse students by comparing the full output but sending back truncated output. 225 | act_output = truncate(act_output) 226 | try: 227 | correct = test.compare_results(exp_output, act_output) 228 | except EndTest as e: 229 | # Allows a grader's compare_results function to raise an EndTest exception 230 | # (defined in gradelib.py). This enables the checker to print out an error 231 | # message to the student, which will be appended to the end of stdout. 232 | if e is not None: 233 | act_output += '\n' 234 | error_msg = _("ERROR") 235 | act_output += "*** {error_msg}: {error_detail} ***".format( 236 | error_msg=error_msg, 237 | error_detail=e 238 | ) 239 | correct = False 240 | corrects.append(correct) 241 | if not grader_config.get("hide_output", False): 242 | results['tests'].append((exp_short_desc, exp_long_desc, 243 | correct, exp_output, act_output)) 244 | 245 | # If there were no tests run, then there was probably an error, so it's incorrect 246 | n = len(corrects) 247 | results['correct'] = all(corrects) and n > 0 248 | results['score'] = float(sum(corrects))/n if n > 0 else 0 249 | 250 | if n == 0 and len(results['errors']) == 0: 251 | results['errors'] = [ 252 | _("There was a problem while running your code (Staff debug: L450). " 253 | "Please contact the course staff for assistance.") 254 | ] 255 | 256 | return results 257 | 258 | 259 | def main(args): # pragma: no cover 260 | """ 261 | Prints a json list: 262 | [ ("Test description", "value") ] 263 | 264 | TODO: what about multi-file submission? 265 | """ 266 | import logging 267 | from pprint import pprint 268 | from codejail.jail_code import configure 269 | import getpass 270 | 271 | logging.basicConfig(level=logging.DEBUG) 272 | if len(args) != 2: 273 | return 274 | 275 | configure("python", sys.executable, user=getpass.getuser()) 276 | (grader_path, submission_path) = args 277 | 278 | with open(submission_path) as f: 279 | submission = f.read().decode('utf-8') 280 | 281 | grader_config = {"lang": "eo"} 282 | grader_path = path(grader_path).abspath() 283 | g = JailedGrader(grader_root=grader_path.dirname().parent.parent) 284 | pprint(g.grade(grader_path, grader_config, submission)) 285 | 286 | 287 | if __name__ == '__main__': # pragma: no cover 288 | main(sys.argv[1:]) 289 | -------------------------------------------------------------------------------- /xqueue_watcher/manager.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import getpass 3 | import importlib 4 | import inspect 5 | import json 6 | import logging 7 | import logging.config 8 | from path import Path 9 | import signal 10 | import sys 11 | import time 12 | 13 | from codejail import jail_code 14 | 15 | from .settings import get_manager_config_values, MANAGER_CONFIG_DEFAULTS 16 | 17 | 18 | class Manager: 19 | """ 20 | Manages polling connections to XQueue. 21 | """ 22 | def __init__(self): 23 | self.clients = [] 24 | self.log = logging 25 | self.manager_config = MANAGER_CONFIG_DEFAULTS.copy() 26 | 27 | def client_from_config(self, queue_name, watcher_config): 28 | """ 29 | Return an XQueueClient from the configuration object. 30 | """ 31 | from . import client 32 | 33 | klass = getattr(client, watcher_config.get('CLASS', 'XQueueClientThread')) 34 | watcher = klass( 35 | queue_name, 36 | xqueue_server=watcher_config.get('SERVER', 'http://localhost:18040'), 37 | xqueue_auth=watcher_config.get('AUTH', (None, None)), 38 | http_basic_auth=self.manager_config['HTTP_BASIC_AUTH'], 39 | requests_timeout=self.manager_config['REQUESTS_TIMEOUT'], 40 | poll_interval=self.manager_config['POLL_INTERVAL'], 41 | login_poll_interval=self.manager_config['LOGIN_POLL_INTERVAL'], 42 | ) 43 | 44 | for handler_config in watcher_config.get('HANDLERS', []): 45 | 46 | handler_name = handler_config['HANDLER'] 47 | mod_name, classname = handler_name.rsplit('.', 1) 48 | module = importlib.import_module(mod_name) 49 | 50 | kw = handler_config.get('KWARGS', {}) 51 | 52 | # codejail configuration per handler 53 | codejail_config = handler_config.get("CODEJAIL", None) 54 | if codejail_config: 55 | kw['codejail_python'] = self.enable_codejail(codejail_config) 56 | try: 57 | handler = getattr(module, classname) 58 | except AttributeError: 59 | if classname == 'urlencode' and mod_name == 'urllib': 60 | module = importlib.import_module('urllib.parse') 61 | handler = getattr(module, classname) 62 | if kw or inspect.isclass(handler): 63 | # handler could be a function or a class 64 | handler = handler(**kw) 65 | watcher.add_handler(handler) 66 | return watcher 67 | 68 | def configure(self, configuration): 69 | """ 70 | Configure XQueue clients. 71 | """ 72 | for queue_name, config in configuration.items(): 73 | for i in range(config.get('CONNECTIONS', 1)): 74 | watcher = self.client_from_config(queue_name, config) 75 | self.clients.append(watcher) 76 | 77 | def configure_from_directory(self, directory): 78 | """ 79 | Load configuration files from the config_root 80 | and one or more queue configurations from a conf.d 81 | directory relative to the config_root 82 | """ 83 | directory = Path(directory) 84 | 85 | log_config = directory / 'logging.json' 86 | if log_config.exists(): 87 | with open(log_config) as config: 88 | logging.config.dictConfig(json.load(config)) 89 | else: 90 | logging.basicConfig(level="DEBUG") 91 | self.log = logging.getLogger('xqueue_watcher.manager') 92 | 93 | app_config_path = directory / 'xqwatcher.json' 94 | self.manager_config = get_manager_config_values(app_config_path) 95 | 96 | confd = directory / 'conf.d' 97 | for watcher in confd.files('*.json'): 98 | with open(watcher) as queue_config: 99 | self.configure(json.load(queue_config)) 100 | 101 | def enable_codejail(self, codejail_config): 102 | """ 103 | Enable codejail for the process. 104 | codejail_config is a dict like this: 105 | { 106 | "name": "python", 107 | "bin_path": "/path/to/python", 108 | "user": "sandbox_username", 109 | "limits": { 110 | "CPU": 1, 111 | ... 112 | } 113 | } 114 | limits are optional 115 | user defaults to the current user 116 | """ 117 | name = codejail_config["name"] 118 | bin_path = codejail_config['bin_path'] 119 | user = codejail_config.get('user', getpass.getuser()) 120 | jail_code.configure(name, bin_path, user=user) 121 | limits = codejail_config.get("limits", {}) 122 | for limit_name, value in limits.items(): 123 | jail_code.set_limit(limit_name, value) 124 | self.log.info("configured codejail -> %s %s %s", name, bin_path, user) 125 | return name 126 | 127 | def start(self): 128 | """ 129 | Start XQueue client threads (or processes). 130 | """ 131 | for c in self.clients: 132 | self.log.info('Starting %r', c) 133 | c.start() 134 | 135 | def wait(self): 136 | """ 137 | Monitor clients. 138 | """ 139 | if not self.clients: 140 | return 141 | signal.signal(signal.SIGTERM, self.shutdown) 142 | while 1: 143 | for client in self.clients: 144 | if not client.is_alive(): 145 | self.log.error('Client died -> %r', 146 | client.queue_name) 147 | self.shutdown() 148 | try: 149 | time.sleep(self.manager_config['POLL_TIME']) 150 | except KeyboardInterrupt: # pragma: no cover 151 | self.shutdown() 152 | 153 | def shutdown(self, *args): 154 | """ 155 | Cleanly shutdown all clients. 156 | """ 157 | self.log.info('shutting down') 158 | while self.clients: 159 | client = self.clients.pop() 160 | client.shutdown() 161 | if client.processing: 162 | try: 163 | client.join() 164 | except RuntimeError: 165 | self.log.exception("joining") 166 | sys.exit(9) 167 | self.log.info('%r done', client) 168 | self.log.info('done') 169 | sys.exit() 170 | 171 | 172 | def main(args=None): 173 | import argparse 174 | parser = argparse.ArgumentParser(prog="xqueue_watcher", description="Run grader from settings") 175 | parser.add_argument('-d', '--config_root', required=True, 176 | help='Configuration root from which to load general ' 177 | 'watcher configuration. Queue configuration ' 178 | 'is loaded from a conf.d directory relative to ' 179 | 'the root') 180 | args = parser.parse_args(args) 181 | 182 | manager = Manager() 183 | manager.configure_from_directory(args.config_root) 184 | 185 | if not manager.clients: 186 | print("No xqueue watchers configured") 187 | manager.start() 188 | manager.wait() 189 | return 0 190 | -------------------------------------------------------------------------------- /xqueue_watcher/settings.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | MANAGER_CONFIG_DEFAULTS = { 4 | 'HTTP_BASIC_AUTH': None, 5 | 'POLL_TIME': 10, 6 | 'REQUESTS_TIMEOUT': 1, 7 | 'POLL_INTERVAL': 1, 8 | 'LOGIN_POLL_INTERVAL': 5, 9 | 'FOLLOW_CLIENT_REDIRECTS': False 10 | } 11 | 12 | 13 | def get_manager_config_values(app_config_path): 14 | if not app_config_path.exists(): 15 | return MANAGER_CONFIG_DEFAULTS.copy() 16 | with open(app_config_path) as config: 17 | config_tokens = json.load(config) 18 | return { 19 | config_key: config_tokens.get(config_key, default_config_value) 20 | for config_key, default_config_value in MANAGER_CONFIG_DEFAULTS.items() 21 | } 22 | --------------------------------------------------------------------------------