├── .gitignore ├── .pylintrc ├── .style.yapf ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── Vagrantfile ├── build ├── .gitignore ├── logo.png ├── logo.svg └── ubuntu-install.sh ├── chomp ├── __init__.py ├── cache.py ├── config.py ├── decompose.py ├── exceptions.py ├── helpers.py ├── shift_collection.py ├── splitter.py └── tasking.py ├── conf ├── nginx-app.conf └── supervisor-app.conf ├── functional-tests ├── __init__.py ├── test_decompose.py └── test_splitter.py ├── makefile ├── requirements.txt ├── server.sh ├── setup.cfg ├── setup.py ├── tests ├── __init__.py ├── test_cache.py ├── test_decompose.py ├── test_helpers.py ├── test_shift_collection.py └── test_splitter.py └── vagrant.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .DS_Store 3 | .vagrant/ 4 | *.log 5 | vagrant-venv/ 6 | .cache/ 7 | *.swp 8 | *.prm 9 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # We need to tell where the python path is 4 | init-hook='import sys;sys.path.append("/src/");sys.path.append("/vagrant/");sys.path.append("/home/ubuntu/chomp/")' 5 | 6 | # Specify a configuration file. 7 | rcfile= 8 | 9 | # Add files or directories to the blacklist. They should be base names, not 10 | # paths. 11 | ignore=CVS 12 | 13 | # Pickle collected data for later comparisons. 14 | persistent=no 15 | 16 | # List of plugins (as comma separated values of python modules names) to load, 17 | # usually to register additional checkers. 18 | load-plugins= 19 | 20 | # Use multiple processes to speed up Pylint. 21 | jobs=1 22 | 23 | # Allow loading of arbitrary C extensions. Extensions are imported into the 24 | # active Python interpreter and may run arbitrary code. 25 | unsafe-load-any-extension=no 26 | 27 | # A comma-separated list of package or module names from where C extensions may 28 | # be loaded. Extensions are loading into the active Python interpreter and may 29 | # run arbitrary code 30 | extension-pkg-whitelist= 31 | 32 | # Allow optimization of some AST trees. This will activate a peephole AST 33 | # optimizer, which will apply various small optimizations. For instance, it can 34 | # be used to obtain the result of joining multiple strings with the addition 35 | # operator. Joining a lot of strings can lead to a maximum recursion error in 36 | # Pylint and this flag can prevent that. It has one side effect, the resulting 37 | # AST will be different than the one from reality. 38 | optimize-ast=no 39 | 40 | 41 | [MESSAGES CONTROL] 42 | 43 | # Only show warnings with the listed confidence levels. Leave empty to show 44 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 45 | confidence= 46 | 47 | # Enable the message, report, category or checker with the given id(s). You can 48 | # either give multiple identifier separated by comma (,) or put this option 49 | # multiple time (only on the command line, not in the configuration file where 50 | # it should appear only once). See also the "--disable" option for examples. 51 | #enable= 52 | 53 | # Disable the message, report, category or checker with the given id(s). You 54 | # can either give multiple identifiers separated by comma (,) or put this 55 | # option multiple times (only on the command line, not in the configuration 56 | # file where it should appear only once).You can also use "--disable=all" to 57 | # disable everything first and then reenable specific checks. For example, if 58 | # you want to run only the similarities checker, you can use "--disable=all 59 | # --enable=similarities". If you want to run only the classes checker, but have 60 | # no Warning level messages displayed, use"--disable=all --enable=classes 61 | # --disable=W" 62 | disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,line-too-long,missing-docstring,invalid-name,no-member,bare-except,no-init,fixme,unused-argument,bad-builtin,deprecated-lambda,broad-except,protected-access,locally-disabled,C,R 63 | 64 | # We want warnings and errors only 65 | # So disable convention and refactor (C and R) 66 | 67 | [REPORTS] 68 | 69 | # Set the output format. Available formats are text, parseable, colorized, msvs 70 | # (visual studio) and html. You can also give a reporter class, eg 71 | # mypackage.mymodule.MyReporterClass. 72 | output-format=text 73 | 74 | # Put messages in a separate file for each module / package specified on the 75 | # command line instead of printing them on stdout. Reports (if any) will be 76 | # written in a file name "pylint_global.[txt|html]". 77 | files-output=no 78 | 79 | # Tells whether to display a full report or only the messages 80 | reports=no 81 | 82 | # Python expression which should return a note less than 10 (10 is the highest 83 | # note). You have access to the variables errors warning, statement which 84 | # respectively contain the number of errors / warnings messages and the total 85 | # number of statements analyzed. This is used by the global evaluation report 86 | # (RP0004). 87 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 88 | 89 | # Template used to display messages. This is a python new-style format string 90 | # used to format the message information. See doc for all details 91 | #msg-template= 92 | 93 | 94 | [BASIC] 95 | 96 | # List of builtins function names that should not be used, separated by a comma 97 | bad-functions=map,filter,input 98 | 99 | # Good variable names which should always be accepted, separated by a comma 100 | good-names=i,j,k,ex,Run,_ 101 | 102 | # Bad variable names which should always be refused, separated by a comma 103 | bad-names=foo,bar,baz,toto,tutu,tata 104 | 105 | # Colon-delimited sets of names that determine each other's naming style when 106 | # the name regexes allow several styles. 107 | name-group= 108 | 109 | # Include a hint for the correct naming format with invalid-name 110 | include-naming-hint=no 111 | 112 | # Regular expression matching correct function names 113 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 114 | 115 | # Naming hint for function names 116 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 117 | 118 | # Regular expression matching correct variable names 119 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 120 | 121 | # Naming hint for variable names 122 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 123 | 124 | # Regular expression matching correct constant names 125 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 126 | 127 | # Naming hint for constant names 128 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 129 | 130 | # Regular expression matching correct attribute names 131 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 132 | 133 | # Naming hint for attribute names 134 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 135 | 136 | # Regular expression matching correct argument names 137 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 138 | 139 | # Naming hint for argument names 140 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 141 | 142 | # Regular expression matching correct class attribute names 143 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 144 | 145 | # Naming hint for class attribute names 146 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 147 | 148 | # Regular expression matching correct inline iteration names 149 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 150 | 151 | # Naming hint for inline iteration names 152 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 153 | 154 | # Regular expression matching correct class names 155 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 156 | 157 | # Naming hint for class names 158 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 159 | 160 | # Regular expression matching correct module names 161 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 162 | 163 | # Naming hint for module names 164 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 165 | 166 | # Regular expression matching correct method names 167 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 168 | 169 | # Naming hint for method names 170 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 171 | 172 | # Regular expression which should only match function or class names that do 173 | # not require a docstring. 174 | no-docstring-rgx=^_ 175 | 176 | # Minimum line length for functions/classes that require docstrings, shorter 177 | # ones are exempt. 178 | docstring-min-length=-1 179 | 180 | 181 | [ELIF] 182 | 183 | # Maximum number of nested blocks for function / method body 184 | max-nested-blocks=5 185 | 186 | 187 | [FORMAT] 188 | 189 | # Maximum number of characters on a single line. 190 | max-line-length=100 191 | 192 | # Regexp for a line that is allowed to be longer than the limit. 193 | ignore-long-lines=^\s*(# )??$ 194 | 195 | # Allow the body of an if to be on the same line as the test if there is no 196 | # else. 197 | single-line-if-stmt=no 198 | 199 | # List of optional constructs for which whitespace checking is disabled. `dict- 200 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 201 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 202 | # `empty-line` allows space-only lines. 203 | no-space-check=trailing-comma,dict-separator 204 | 205 | # Maximum number of lines in a module 206 | max-module-lines=1000 207 | 208 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 209 | # tab). 210 | indent-string=' ' 211 | 212 | # Number of spaces of indent required inside a hanging or continued line. 213 | indent-after-paren=4 214 | 215 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 216 | expected-line-ending-format= 217 | 218 | 219 | [LOGGING] 220 | 221 | # Logging modules to check that the string format arguments are in logging 222 | # function parameter format 223 | logging-modules=logging 224 | 225 | 226 | [MISCELLANEOUS] 227 | 228 | # List of note tags to take in consideration, separated by a comma. 229 | notes=FIXME,XXX,TODO 230 | 231 | 232 | [SIMILARITIES] 233 | 234 | # Minimum lines number of a similarity. 235 | min-similarity-lines=4 236 | 237 | # Ignore comments when computing similarities. 238 | ignore-comments=yes 239 | 240 | # Ignore docstrings when computing similarities. 241 | ignore-docstrings=yes 242 | 243 | # Ignore imports when computing similarities. 244 | ignore-imports=no 245 | 246 | 247 | [SPELLING] 248 | 249 | # Spelling dictionary name. Available dictionaries: none. To make it working 250 | # install python-enchant package. 251 | spelling-dict= 252 | 253 | # List of comma separated words that should not be checked. 254 | spelling-ignore-words= 255 | 256 | # A path to a file that contains private dictionary; one word per line. 257 | spelling-private-dict-file= 258 | 259 | # Tells whether to store unknown words to indicated private dictionary in 260 | # --spelling-private-dict-file option instead of raising a message. 261 | spelling-store-unknown-words=no 262 | 263 | 264 | [TYPECHECK] 265 | 266 | # Tells whether missing members accessed in mixin class should be ignored. A 267 | # mixin class is detected if its name ends with "mixin" (case insensitive). 268 | ignore-mixin-members=yes 269 | 270 | # List of module names for which member attributes should not be checked 271 | # (useful for modules/projects where namespaces are manipulated during runtime 272 | # and thus existing member attributes cannot be deduced by static analysis. It 273 | # supports qualified module names, as well as Unix pattern matching. 274 | ignored-modules= 275 | 276 | # List of classes names for which member attributes should not be checked 277 | # (useful for classes with attributes dynamically set). This supports can work 278 | # with qualified names. 279 | ignored-classes=LookupDict,jinja_env,tuple 280 | 281 | # List of members which are set dynamically and missed by pylint inference 282 | # system, and so shouldn't trigger E1101 when accessed. Python regular 283 | # expressions are accepted. 284 | generated-members=query 285 | 286 | 287 | [VARIABLES] 288 | 289 | # Tells whether we should check for unused import in __init__ files. 290 | init-import=no 291 | 292 | # A regular expression matching the name of dummy variables (i.e. expectedly 293 | # not used). 294 | dummy-variables-rgx=_$|dummy 295 | 296 | # List of additional names supposed to be defined in builtins. Remember that 297 | # you should avoid to define new builtins when possible. 298 | additional-builtins= 299 | 300 | # List of strings which can identify a callback function by name. A callback 301 | # name must start or end with one of those strings. 302 | callbacks=cb_,_cb 303 | 304 | 305 | [CLASSES] 306 | 307 | # List of method names used to declare (i.e. assign) instance attributes. 308 | defining-attr-methods=__init__,__new__,setUp 309 | 310 | # List of valid names for the first argument in a class method. 311 | valid-classmethod-first-arg=cls 312 | 313 | # List of valid names for the first argument in a metaclass class method. 314 | valid-metaclass-classmethod-first-arg=mcs 315 | 316 | # List of member names, which should be excluded from the protected access 317 | # warning. 318 | exclude-protected=_asdict,_fields,_replace,_source,_make 319 | 320 | 321 | [DESIGN] 322 | 323 | # Maximum number of arguments for function / method 324 | max-args=5 325 | 326 | # Argument names that match this expression will be ignored. Default to name 327 | # with leading underscore 328 | ignored-argument-names=_.* 329 | 330 | # Maximum number of locals for function / method body 331 | max-locals=15 332 | 333 | # Maximum number of return / yield for function / method body 334 | max-returns=6 335 | 336 | # Maximum number of branch for function / method body 337 | max-branches=12 338 | 339 | # Maximum number of statements in function / method body 340 | max-statements=50 341 | 342 | # Maximum number of parents for a class (see R0901). 343 | max-parents=7 344 | 345 | # Maximum number of attributes for a class (see R0902). 346 | max-attributes=7 347 | 348 | # Minimum number of public methods for a class (see R0903). 349 | min-public-methods=2 350 | 351 | # Maximum number of public methods for a class (see R0904). 352 | max-public-methods=20 353 | 354 | # Maximum number of boolean expressions in a if statement 355 | max-bool-expr=5 356 | 357 | 358 | [IMPORTS] 359 | 360 | # Deprecated modules which should not be used, separated by a comma 361 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 362 | 363 | # Create a graph of every (i.e. internal and external) dependencies in the 364 | # given file (report RP0402 must not be disabled) 365 | import-graph= 366 | 367 | # Create a graph of external dependencies in the given file (report RP0402 must 368 | # not be disabled) 369 | ext-import-graph= 370 | 371 | # Create a graph of internal dependencies in the given file (report RP0402 must 372 | # not be disabled) 373 | int-import-graph= 374 | 375 | 376 | [EXCEPTIONS] 377 | 378 | # Exceptions that will emit a warning when being caught. Defaults to 379 | # "Exception" 380 | overgeneral-exceptions=Exception 381 | -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = pep8 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | sudo: required 3 | language: python 4 | python: 5 | - '2.7' 6 | services: 7 | - memcached 8 | env: 9 | global: 10 | - ENV=test 11 | - PYTHONPATH="$TRAVIS_BUILD_DIR" 12 | install: 13 | - cd build && bash ubuntu-install.sh && cd .. 14 | - pip install -I -r requirements.txt 15 | script: make test 16 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04 2 | ENV DEBIAN_FRONTEND noninteractive 3 | 4 | # Environment Variables 5 | ENV PYTHONPATH "/src" 6 | ENV ENV test 7 | 8 | 9 | # setup tools 10 | RUN apt-get update --yes --force-yes 11 | RUN apt-get install --yes --force-yes build-essential python python-setuptools curl python-pip libssl-dev 12 | RUN apt-get update --yes --force-yes 13 | RUN apt-get install --yes --force-yes python-software-properties libffi-dev libssl-dev python-dev 14 | 15 | RUN apt-get install --yes --force-yes nginx supervisor memcached 16 | 17 | # Add and install Python modules 18 | ADD requirements.txt /src/requirements.txt 19 | RUN cd /src; pip install -r requirements.txt 20 | 21 | # Bundle app source 22 | ADD . /src 23 | 24 | # configuration 25 | RUN echo "daemon off;" >> /etc/nginx/nginx.conf 26 | RUN rm /etc/nginx/sites-enabled/default 27 | RUN ln -s /src/conf/nginx-app.conf /etc/nginx/sites-enabled/ 28 | RUN ln -s /src/conf/supervisor-app.conf /etc/supervisor/conf.d/ 29 | RUN cd /src/ && make build 30 | 31 | # Expose - note that load balancer terminates SSL 32 | EXPOSE 80 33 | 34 | # RUN 35 | CMD ["supervisord", "-n"] 36 | 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2015-2017 StaffJoy, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Logo](https://i.imgur.com/deZ3wCa.jpg) 2 | 3 | # Chomp - Service for computing shifts from forecasts 4 | 5 | [![Build Status](https://travis-ci.org/Staffjoy/chomp-decomposition.svg?branch=master)](https://travis-ci.org/Staffjoy/chomp-decomposition) [![Moonlight contractors](https://www.moonlightwork.com/shields/python.svg)](https://www.moonlightwork.com/for/python?referredByUserID=1&referralProgram=maintainer&referrerName=Staffjoy) [![Docker Automated build](https://img.shields.io/docker/automated/jrottenberg/ffmpeg.svg)](https://hub.docker.com/r/staffjoy/chomp-decomposition/) [![PyPI version](https://badge.fury.io/py/chomp.svg)](https://badge.fury.io/py/chomp) 6 | 7 | [Staffjoy is shutting down](https://blog.staffjoy.com/staffjoy-is-shutting-down-39f7b5d66ef6#.ldsdqb1kp), so we are open-sourcing our code. Chomp is an applied mathematics microservice for decomposing hourly demand into shifts of variable length. This repo was intended to be a proof of concept. It worked so well in production that we never rewrote it. My intention was to rewrite it into a more parallel language, such as Go, in order to take advantage of multiple cores. It served production traffic from June 2016 to March 2017 with zero modification or production errors. 8 | 9 | ## Self-Hosting 10 | 11 | If you are self-hosting Chomp, please make sure that you are using version `>=0.24` of the `staffoy` client (in `requirements.txt`). Then, modify `chomp/tasking.py` to include your custom domain - when initializing `Client()`, pass in your custom URL base, e.g. `Client(url_base="https://staffjoy.example.com/api/v2/, key=config.STAFFJOY_API_KEY, env=config.ENV)`, 12 | 13 | ## Algorithm 14 | 15 | ![Chomp converts forecasts to shifts](https://i.imgur.com/i8enKgO.png) 16 | 17 | Chomp solves a type of packing problem similar to a [bin packing problem](https://en.wikipedia.org/wiki/Bin_packing_problem). It tries to tessellate when shifts start and how long they last in order to best match staffing levels to forecasts. It does this subject to the minimum and maximum shift length. 18 | 19 | Chomp uses techniques from [branch and bound algorithms](https://en.wikipedia.org/wiki/Branch_and_bound), and adds in subproblem generation, preprocessing techniques, feasibility detection, heuristics, and caching. [Learn more in the Chomp launch blog post.](https://blog.staffjoy.com/introducing-chomp-computing-shifts-from-forecasts-21315f46aadc#.xa306ltre) 20 | 21 | 22 | ## Credit 23 | 24 | This repository was conceived and authored in its entirety by [@philipithomas](https://github.com/philipithomas). This is a fork of the internal repository. For security purposes, the Git history has been squashed and client names have been scrubbed from tests. (Whenever there was a production issue, we added a functional test to the repo.) 25 | 26 | ## Usage 27 | 28 | You can install Chomp locally with `pip install --upgrade chomp`. Or, use the [docker image](https://hub.docker.com/r/staffjoy/chomp-decomposition/) to interface Chomp with [Staffjoy Suite](https://github.com/staffjoy/suite). 29 | 30 | 31 | Chomp was designed to handle one week of scheduling at an hourly granularity. However, the package is unitless and hypothetically support weeks of any length of time at any granularity. 32 | 33 | ``` 34 | from chomp import Decompose 35 | 36 | demand = [0, 0, 0, 0, 0, 0, 0, 5, 5, 7, 8, 6, 6, 7, 7, 7, 9, 9, 6, 5, 37 | 4, 4, 0, 0] 38 | min_length = 4 39 | max_length = 8 40 | d = Decompose(demand, min_length, max_length) 41 | d.calculate() # Long step - runs calculation 42 | d.validate() # Mainly for testing, but quick compared to calculate step. Verifies that solution meets demand. 43 | d.get_shifts() # Returns a list of shifts, each of which have a start time and length 44 | # [{'start': 7, 'length': 4}, {'start': 7, 'length': 4}, {'start': 7, 'length': 4}, {'start': 7, 'length': 4}, {'start': 7, 'length': 4}, {'start': 9, 'length': 4}, {'start': 9, 'length': 4}, {'start': 10, 'length': 4}, {'start': 11, 'length': 4}, {'start': 11, 'length': 4}, {'start': 11, 'length': 4}, {'start': 13, 'length': 4}, {'start': 13, 'length': 5}, {'start': 13, 'length': 5}, {'start': 14, 'length': 4}, {'start': 15, 'length': 4}, {'start': 15, 'length': 5}, {'start': 15, 'length': 7}, {'start': 16, 'length': 6}, {'start': 16, 'length': 6}, {'start': 17, 'length': 5}] 45 | ``` 46 | 47 | Chomp seeks to minimize the time worked, i.e. `sum([shift.length for shift in shifts])`, where `min_length <= shift.length <= max_length` for every shift. 48 | ## Environment Variables 49 | 50 | This table intends to explain the main requirements specified in `app/config.py`. This configuration file can be manually edited, but be careful to not commit secret information into the git repository. Please explore the config file for full customization info. 51 | 52 | Name | Description | Example Format 53 | ---- | ----------- | -------------- 54 | ENV | "prod", "stage", or "dev" to specify the configuration to use. When running the code, use "prod". | prod 55 | STAFFJOY_API_KEY | Api key for accessing the Staffjoy API that has at least `sudo` permission level | 56 | SYSLOG_SERVER | host and port for a syslog server, e.g. [papertrailapp.com](http://papertrailapp.com) | logs2.papertrailapp.com:12345 57 | 58 | ## Running 59 | 60 | Provision the machine with vagrant. When you first run the program or when you change `requirements.txt`, run `make requirements` to install and freeze the required libraries. 61 | 62 | ``` 63 | vagrant up 64 | vagrant ssh 65 | # (In VM) 66 | cd /vagrant/ 67 | make dependencies 68 | ``` 69 | 70 | ## Caching 71 | 72 | Subproblems are cached based on their demand, minimum shift length, and maximum shift length. This prevents re-calculation of problems whose answer we know. Currently this cache lives on the box in Memcache. Clearly, this means that a deploy, restart, etc can trigger a loss of all historical data. For now, this is by design so that theroetical efficiency gains by newer builds can be realized. In the future, we may want to tag things that are at perfect optimality and preserve them by using a dedicated memcache cluster. Realistically though, most repeated problems will be within the same "week" by orgs that repeat demand for all weekdays or the like. 73 | 74 | ## Formatting 75 | 76 | This library uses the [Google YAPF](https://github.com/google/yapf) library to enforce PEP-8. Using it is easy - run `make fmt` to format your code inline correctly. Failure to do this will result in your build failing. You have been warned. 77 | 78 | To disable YAPf around code that you do not want changed, wrap it like this: 79 | 80 | ``` 81 | # yapf: disable 82 | FOO = { 83 | # ... some very large, complex data literal. 84 | } 85 | 86 | BAR = [ 87 | # ... another large data literal. 88 | ] 89 | # yapf: enable 90 | ``` 91 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | VAGRANTFILE_API_VERSION = "2" 5 | Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| 6 | config.vm.box = "trusty-amd64" 7 | config.vm.box_url = "http://cloud-images.ubuntu.com/vagrant/trusty/current/trusty-server-cloudimg-amd64-vagrant-disk1.box" 8 | config.vm.network :private_network, ip: "192.168.69.97" 9 | config.vm.synced_folder ".", "/vagrant" 10 | config.vm.hostname = "chomp-dev.staffjoy.com" 11 | config.vm.provider :virtualbox do |vb| 12 | vb.customize ["modifyvm", :id, "--memory", "2000"] 13 | vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"] 14 | end 15 | 16 | config.vm.provision "shell", path: "vagrant.sh", privileged: false 17 | end 18 | -------------------------------------------------------------------------------- /build/.gitignore: -------------------------------------------------------------------------------- 1 | gurobi650/ 2 | gurobi605/ 3 | -------------------------------------------------------------------------------- /build/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Staffjoy/chomp-decomposition/871729063176679c07c4a4b91f39d0fbcf7b1737/build/logo.png -------------------------------------------------------------------------------- /build/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 20 | 21 | 22 | 23 | 24 | 25 | 29 | 30 | 31 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /build/ubuntu-install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | sudo apt-get update --yes --force-yes 5 | 6 | # Sometimes ubuntu needs these software-properties repos :-( 7 | sudo apt-get install --yes --force-yes build-essential libffi-dev libssl-dev cmake expect-dev \ 8 | python python-dev curl python-setuptools python-software-properties 9 | 10 | # For now, we are putting memcache in dev, stage, and prod 11 | # (including inside the docker container) 12 | sudo apt-get install --yes memcached 13 | 14 | sudo easy_install -U pip 15 | 16 | sudo apt-get update --yes --force-yes # Re-update 17 | 18 | # Set env variable that we are in dev 19 | echo "echo 'export env=\"dev\"' >> /etc/profile" | sudo bash 20 | -------------------------------------------------------------------------------- /chomp/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from logging.handlers import SysLogHandler 3 | import sys 4 | import os 5 | 6 | from .config import config 7 | from .cache import Cache 8 | 9 | # This file initializes the Chomp package. 10 | # 11 | # Within the chomp files, you want to import these: 12 | # 13 | # * logger - for logging 14 | # * config - for getting different configurations 15 | 16 | # Now when things import, we load the settings based on their env 17 | config = config[os.environ.get("ENV", "dev")] 18 | 19 | # Logging configuration 20 | logger = logging.getLogger(__name__) 21 | 22 | 23 | class ContextFilter(logging.Filter): 24 | hostname = "chomp-%s" % config.ENV 25 | 26 | def filter(self, record): 27 | record.hostname = self.hostname 28 | return True 29 | 30 | 31 | f = ContextFilter() 32 | logger.setLevel(config.LOG_LEVEL) 33 | logger.addFilter(f) 34 | 35 | if config.SYSLOG: 36 | # Send to syslog / papertrail server 37 | syslog_tuple = config.SYSLOG_SERVER.split(":") 38 | handler = SysLogHandler(address=(syslog_tuple[0], int(syslog_tuple[1]))) 39 | else: 40 | # Just print to standard out 41 | handler = logging.StreamHandler(sys.stdout) 42 | 43 | formatter = logging.Formatter( 44 | '%(asctime)s %(hostname)s chomp %(levelname)s %(message)s', 45 | datefmt='%Y-%m-%dT%H:%M:%S') 46 | handler.setFormatter(formatter) 47 | handler.setLevel(config.LOG_LEVEL) 48 | logger.addHandler(handler) 49 | 50 | # Set up caching client 51 | cache = Cache(config, logger) 52 | 53 | # Import things we are exporting 54 | from .decompose import Decompose 55 | from .splitter import Splitter 56 | from .tasking import Tasking 57 | 58 | logger.info("Initialized environment %s", config.ENV) 59 | -------------------------------------------------------------------------------- /chomp/cache.py: -------------------------------------------------------------------------------- 1 | import memcache 2 | import json 3 | import hashlib 4 | 5 | 6 | class Cache(): 7 | """Subproblem caching""" 8 | 9 | def __init__(self, config, logger): 10 | self.mc = memcache.Client(config.MEMCACHED_CONFIG) 11 | self.config = config 12 | self.logger = logger 13 | 14 | def set(self, shifts=None, **subproblem): 15 | """Given a subproblem's inputs, store the results""" 16 | if shifts is None or len(shifts) is 0: 17 | raise Exception("Do not set an empty cache") 18 | 19 | self.mc.set(self._subproblem_to_key(subproblem), shifts) 20 | 21 | def get(self, **subproblem): 22 | """Check cache for a subproblem""" 23 | return self.mc.get(self._subproblem_to_key(subproblem)) 24 | 25 | def flush(self): 26 | """Flush all caches. Mainly used for testing.""" 27 | # Set to warning becuase this probably shouldn't happen in prod 28 | self.logger.warning("Cache flushed") 29 | self.mc.flush_all() 30 | 31 | @staticmethod 32 | def _subproblem_to_key(subproblem): 33 | """Convert a subproblem into a memcached key""" 34 | # Cannot just use straight json because it has spaces, 35 | # which memcache does not support. Hash is repeatable and 36 | # also make sure we stay under the memcache key length limit 37 | return hashlib.sha256( 38 | json.dumps(subproblem, sort_keys=True)).hexdigest() 39 | -------------------------------------------------------------------------------- /chomp/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | basedir = os.path.abspath(os.path.dirname(__file__)) 5 | 6 | 7 | class DefaultConfig: 8 | ENV = "prod" 9 | LOG_LEVEL = logging.INFO 10 | SYSLOG = True # Send logs to syslog server 11 | # Logging - we use papertrail.com 12 | SYSLOG_SERVER = os.getenv("SYSLOG_SERVER") 13 | CALCULATION_TIMEOUT = 10 * 60 # 10 minutes, in seconds 14 | BIFURCATION_THRESHHOLD = 100 # Sum of demand needed before splitting 15 | 16 | # Scheduling constants 17 | DAYS_OF_WEEK = [ 18 | "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", 19 | "sunday" 20 | ] 21 | 22 | MEMCACHED_CONFIG = ['127.0.0.1:11211' 23 | ] # Localhost. May centralize in future. 24 | 25 | TASKING_FETCH_INTERVAL_SECONDS = 20 26 | STAFFJOY_API_KEY = os.environ.get("STAFFJOY_API_KEY") 27 | DEFAULT_TZ = "utc" 28 | 29 | # Used for searching for existing shifts 30 | MAX_SHIFT_LENGTH_HOURS = 23 31 | 32 | # Destroy container if there was an error 33 | KILL_ON_ERROR = True 34 | KILL_DELAY = 60 # To prevent infinite loop, sleep before kill 35 | 36 | 37 | class StageConfig(DefaultConfig): 38 | ENV = "stage" 39 | 40 | 41 | class DevelopmentConfig(DefaultConfig): 42 | ENV = "dev" 43 | LOG_LEVEL = logging.DEBUG 44 | SYSLOG = False 45 | TASKING_FETCH_INTERVAL_SECONDS = 5 46 | STAFFJOY_API_KEY = "staffjoydev" 47 | MAX_TUNING_TIME = 5 * 60 # 5 minutes 48 | THREADS = 2 49 | CALCULATION_TIMEOUT = 5 * 60 # 5 minutes, in seconds 50 | KILL_ON_ERROR = False 51 | 52 | 53 | class TestConfig(DefaultConfig): 54 | ENV = "test" 55 | SYSLOG = False 56 | LOG_LEVEL = logging.DEBUG 57 | THREADS = 6 58 | CALCULATION_TIMEOUT = 5 * 60 # 5 minutes, in seconds 59 | KILL_ON_ERROR = False 60 | 61 | 62 | config = { # Determined in main.py 63 | "test": TestConfig, 64 | "dev": DevelopmentConfig, 65 | "stage": StageConfig, 66 | "prod": DefaultConfig, 67 | } 68 | -------------------------------------------------------------------------------- /chomp/decompose.py: -------------------------------------------------------------------------------- 1 | import math 2 | import copy 3 | from copy import deepcopy 4 | from datetime import datetime, timedelta 5 | 6 | from chomp import logger, cache, config 7 | from chomp.helpers import reverse_inclusive_range 8 | from chomp.shift_collection import ShiftCollection 9 | 10 | 11 | class Decompose: 12 | """Class for decomposing demand into shifts""" 13 | 14 | def __init__(self, demand, min_length, max_length, window_offset=0): 15 | self.demand = demand # Set this raw value for testing purposes 16 | self.min_length = min_length 17 | self.max_length = max_length 18 | self.window_offset = window_offset 19 | self._process_demand() # This is the demand used for calculations 20 | 21 | # Preface with underscore bc this should never be accessed directly 22 | # - instead use get_shifts() to apply offset 23 | self._shifts = [] 24 | 25 | def _process_demand(self): 26 | """Apply windowing to demand""" 27 | 28 | demand = copy.copy(self.demand) 29 | 30 | # 1) Remove any lagging zeros. This affects nothing. 31 | while demand[-1] is 0: 32 | demand.pop() # remove last element 33 | 34 | # 2) Remove any leading zeros andtrack with offset 35 | offset = 0 36 | while demand[0] is 0: 37 | demand.pop(0) # remove first element 38 | offset += 1 39 | 40 | # TODO - edge smoothing algorithms 41 | 42 | # Smooth beginning edge 43 | # (TODO - search past to max_length and manually strip out shifts) 44 | peak = 0 45 | for t in range(self.min_length): 46 | if demand[t] > peak: 47 | peak = demand[t] 48 | elif demand[t] < peak: 49 | demand[t] = peak 50 | 51 | peak = 0 52 | for t in reversed( 53 | range((len(demand) - self.min_length - 1), len(demand))): 54 | if demand[t] > peak: 55 | peak = demand[t] 56 | elif demand[t] < peak: 57 | demand[t] = peak 58 | 59 | self.demand = demand 60 | self.window_offset += offset 61 | 62 | logger.debug("Windowing removed %s leading zeros", offset) 63 | logger.debug("Processed demand: %s", self.demand) 64 | 65 | def _split_demand(self, round_up=True): 66 | """Return unprocessed demand in half for subproblems""" 67 | halfsies = [] 68 | for val in self.demand: 69 | half = val / 2.0 70 | if round_up: 71 | # Round up 72 | half = math.ceil(half) 73 | else: 74 | # Round down 75 | half = math.floor(half) 76 | 77 | # Round to avoid weird floating point issues (e.g. 7.999999) 78 | halfsies.append(int(round(half))) 79 | 80 | return halfsies 81 | 82 | def get_shifts(self): 83 | """Return de-windowed shifts""" 84 | shifts = copy.copy(self._shifts) 85 | for shift in shifts: 86 | shift["start"] += self.window_offset 87 | 88 | return shifts 89 | 90 | def validate(self): 91 | """Check whether shifts meet demand. Used in testing.""" 92 | expected_demand = [0] * len(self.demand) 93 | sum_demand = copy.copy(expected_demand) 94 | 95 | logger.debug("Starting validation of %s shifts", len(self._shifts)) 96 | 97 | for shift in self._shifts: 98 | for t in range(shift["start"], shift["start"] + shift["length"]): 99 | # Remove window 100 | sum_demand[t] += 1 101 | 102 | logger.debug("Expected demand: %s", expected_demand) 103 | logger.debug("Scheduled supply: %s", sum_demand) 104 | for t in range(len(expected_demand)): 105 | if sum_demand[t] < expected_demand[t]: 106 | logger.error( 107 | "Demand not met at time %s (demand %s, supply %s)", 108 | t + self.window_offset, expected_demand[t], sum_demand[t]) 109 | raise Exception("Demand not met at time %s" % t) 110 | return True 111 | 112 | def efficiency(self, shifts=None): 113 | """Return the overage as a float. 0 is perfect.""" 114 | if shifts is None: 115 | shifts = self._shifts 116 | 117 | efficiency = (1.0 * sum(shift["length"] 118 | for shift in shifts) / sum(self.demand)) - 1 119 | 120 | return efficiency 121 | 122 | def _set_cache(self): 123 | cache.set( 124 | demand=self.demand, 125 | min_length=self.min_length, 126 | max_length=self.max_length, 127 | shifts=self._shifts) 128 | 129 | def calculate(self): 130 | if len(self._shifts) > 0: 131 | raise Exception("Shifts already calculated") 132 | 133 | # Try checking cache. Putting the check here means it even works for 134 | # subproblems! 135 | cached_shifts = cache.get( 136 | demand=self.demand, 137 | min_length=self.min_length, 138 | max_length=self.max_length) 139 | if cached_shifts: 140 | logger.info("Hit cache") 141 | self._shifts = cached_shifts 142 | return 143 | 144 | # Subproblem splitting 145 | demand_sum = sum(self.demand) 146 | if demand_sum > config.BIFURCATION_THRESHHOLD: 147 | # Subproblems. Split into round up and round down. 148 | logger.info("Initiating split (demand sum %s, threshhold %s)", 149 | demand_sum, config.BIFURCATION_THRESHHOLD) 150 | # Show parent demand sum becuase it can recursively split 151 | demand_up = self._split_demand(round_up=True) 152 | demand_low = self._split_demand(round_up=False) 153 | 154 | d_up = Decompose(demand_up, self.min_length, self.max_length) 155 | d_low = Decompose(demand_low, self.min_length, self.max_length) 156 | 157 | logger.info( 158 | "Beginning upper round subproblem (parent demand sum: %s)", 159 | demand_sum) 160 | d_up.calculate() 161 | 162 | logger.info( 163 | "Beginning lower round subproblem (parent demand sum: %s)", 164 | demand_sum) 165 | d_low.calculate() 166 | 167 | self._shifts.extend(d_up.get_shifts()) 168 | self._shifts.extend(d_low.get_shifts()) 169 | self._set_cache() # Set cache for the parent problem too! 170 | return 171 | 172 | self._calculate() 173 | self._set_cache() 174 | 175 | def _calculate(self): 176 | """Search that tree""" 177 | # Not only do we want optimality, but we want it with 178 | # longest shifts possible. That's why we do DFS on long shifts. 179 | 180 | starting_solution = self.use_heuristics_to_generate_some_solution() 181 | 182 | # Helper variables for branch and bound 183 | best_known_coverage = starting_solution.coverage_sum 184 | best_known_solution = starting_solution 185 | best_possible_solution = sum(self.demand) 186 | 187 | logger.debug("Starting with known coverage %s vs best possible %s", 188 | best_known_coverage, best_possible_solution) 189 | 190 | # Branches to search 191 | # (We want shortest shifts retrieved first, so 192 | # we add shortest and pop() to git last in) 193 | # (a LIFO queue using pop is more efficient in python 194 | # than a FIFO queue using pop(0)) 195 | 196 | stack = [] 197 | 198 | logger.info("Demand: %s", self.demand) 199 | empty_collection = ShiftCollection( 200 | self.min_length, self.max_length, demand=self.demand) 201 | stack.append(empty_collection) 202 | 203 | start_time = datetime.utcnow() 204 | 205 | while len(stack) != 0: 206 | if start_time + timedelta( 207 | seconds=config.CALCULATION_TIMEOUT) < datetime.utcnow(): 208 | logger.info("Exited due to timeout (%s seconds)", 209 | (datetime.utcnow() - start_time).total_seconds()) 210 | break 211 | 212 | # Get a branch 213 | working_collection = stack.pop() 214 | 215 | if working_collection.is_optimal: 216 | # We have a complete solution 217 | logger.info("Found an optimal collection. Exiting.") 218 | self.set_shift_collection_as_optimal(working_collection) 219 | return 220 | 221 | if working_collection.demand_is_met: 222 | if working_collection.coverage_sum < best_known_coverage: 223 | logger.info( 224 | "Better solution found (previous coverage %s / new coverage %s / best_possible %s)", 225 | best_known_coverage, working_collection.coverage_sum, 226 | best_possible_solution) 227 | 228 | # Set new best possible solution 229 | best_known_solution = working_collection 230 | best_known_coverage = working_collection.coverage_sum 231 | else: 232 | logger.debug("Found less optimal solution - continuing") 233 | # discard 234 | del working_collection 235 | 236 | else: 237 | 238 | # New branch to explore - else discard 239 | if working_collection.best_possible_coverage < best_known_coverage: 240 | # Gotta add more shifts! 241 | t = working_collection.get_first_time_demand_not_met() 242 | 243 | # Get shift start time 244 | start = t 245 | for length in reverse_inclusive_range(self.min_length, 246 | self.max_length): 247 | # Make sure we aren't off edge 248 | end_index = start + length 249 | 250 | # Our edge smoothing means this will always work 251 | if end_index <= len(self.demand): 252 | shift = (start, length) 253 | new_collection = deepcopy(working_collection) 254 | new_collection.add_shift(shift) 255 | 256 | if new_collection.demand_is_met: 257 | new_collection.anneal() 258 | 259 | if new_collection.best_possible_coverage < best_known_coverage: 260 | 261 | # Only save it if it's an improvement 262 | stack.append(new_collection) 263 | 264 | self.set_shift_collection_as_optimal(best_known_solution) 265 | 266 | def use_heuristics_to_generate_some_solution(self): 267 | """Use heuristics to generate some feasible solution.""" 268 | # (Used for branch and bound) 269 | 270 | # Heuristic: Fill in only shifts of smallest shift len 271 | 272 | collection = ShiftCollection( 273 | self.min_length, self.max_length, demand=self.demand) 274 | 275 | # Add shifts for the end 276 | 277 | length = self.min_length 278 | start = len(self.demand) - length 279 | end_shift = (start, length) 280 | for _ in range(self.demand[-1]): 281 | collection.add_shift(end_shift) 282 | 283 | # And now, go through time and add shifts 284 | for t in range(len(self.demand)): 285 | delta = collection.get_demand_minus_coverage(t) 286 | 287 | start = t 288 | length = self.min_length 289 | 290 | # On short problems we can exceed end 291 | if start + length > len(self.demand): 292 | start = len(self.demand) - length 293 | 294 | shift = (start, length) 295 | 296 | for _ in range(delta): 297 | collection.add_shift(shift) 298 | 299 | if not collection.demand_is_met: 300 | raise Exception("Heuristic for finding demand failed") 301 | 302 | return collection 303 | 304 | def set_shift_collection_as_optimal(self, collection): 305 | """Update decomposition model with our results""" 306 | # We need to unpack the collection shift models to the 307 | # external shifts model 308 | for shift in collection.shifts: 309 | start, length = shift 310 | self._shifts.append({ 311 | "start": start, 312 | "length": length, 313 | }) 314 | self._set_cache() 315 | -------------------------------------------------------------------------------- /chomp/exceptions.py: -------------------------------------------------------------------------------- 1 | class CalculationException(Exception): 2 | def __init__(self, *args, **kwargs): 3 | Exception.__init__(self, *args, **kwargs) 4 | 5 | 6 | class UnequalDayLengthException(Exception): 7 | def __init__(self, *args, **kwargs): 8 | Exception.__init__(self, *args, **kwargs) 9 | -------------------------------------------------------------------------------- /chomp/helpers.py: -------------------------------------------------------------------------------- 1 | DAYS_OF_WEEK = [ 2 | "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", 3 | "sunday" 4 | ] 5 | 6 | 7 | def week_day_range(start_day="monday"): 8 | """ Return list of days of week in order from start day """ 9 | start_index = DAYS_OF_WEEK.index(start_day) 10 | return DAYS_OF_WEEK[start_index:] + DAYS_OF_WEEK[:start_index] 11 | 12 | 13 | def normalize_to_midnight(dt_obj): 14 | """Take a datetime and round it to midnight""" 15 | return dt_obj.replace(hour=0, minute=0, second=0, microsecond=0) 16 | 17 | 18 | def inclusive_range(start, end): 19 | """Get start to end, inclusive to inclusive""" 20 | # Note that python range is inclusive to exclusive 21 | return range(start, end + 1) 22 | 23 | 24 | def reverse_inclusive_range(low, high): 25 | """Get decreasing range from high to low, inclusive to inclusive""" 26 | # Note that python range is inclusive to exclusive 27 | return range(high, low - 1, -1) 28 | -------------------------------------------------------------------------------- /chomp/shift_collection.py: -------------------------------------------------------------------------------- 1 | from chomp import logger 2 | 3 | 4 | class ShiftCollection(object): 5 | """A group of shifts""" 6 | 7 | def __init__(self, min_length, max_length, demand=None, shifts=None): 8 | if demand is None: 9 | demand = [] 10 | 11 | if shifts is None: 12 | shifts = [] 13 | self._demand = demand 14 | self.demand_length = len(self._demand) 15 | self._coverage = [0] * self.demand_length 16 | 17 | # Used for annealing 18 | self.min_length = min_length 19 | self.max_length = max_length 20 | 21 | # initiate as empty 22 | self._shifts = [] 23 | 24 | # Add in shifts so coverage cache is populated 25 | for shift in shifts: 26 | self.add_shift(shift) 27 | 28 | @property 29 | def shifts(self): 30 | return self._shifts 31 | 32 | @shifts.setter 33 | def shifts(self): 34 | raise Exception("use addShift method to add a shift") 35 | 36 | def add_shift(self, shift): 37 | """Add a shift, which is a tuple of start and length""" 38 | start, length = shift 39 | end_index = start + length 40 | if start < 0 or end_index > self.demand_length: 41 | raise Exception( 42 | "Shift lies outside demand bounds (demand length %s, shift start %s, shift end %s,", 43 | self.demand_length, start, end_index) 44 | 45 | for t in range(start, end_index): 46 | self._coverage[t] += 1 47 | 48 | self._shifts.append(shift) 49 | 50 | def get_demand_minus_coverage(self, t): 51 | """Return needs vs. shift coverage at time""" 52 | # If > 0 then underscheduled 53 | # if = 0 then optimal 54 | # if < 0 then overscheduled 55 | return self._demand[t] - self._coverage[t] 56 | 57 | @property 58 | def coverage_sum(self): 59 | return sum(self._coverage) 60 | 61 | @property 62 | def best_possible_coverage(self): 63 | """Look at current shift + demand needing to be covered""" 64 | best_possible = sum(self._demand) 65 | for t in range(self.demand_length): 66 | delta = self.get_demand_minus_coverage(t) 67 | if delta < 0: 68 | best_possible += abs(delta) 69 | 70 | return best_possible 71 | 72 | @property 73 | def demand_is_met(self): 74 | """Has a solution been found?""" 75 | for t in range(self.demand_length): 76 | if self.get_demand_minus_coverage(t) > 0: 77 | return False 78 | 79 | return True 80 | 81 | @property 82 | def shift_count(self): 83 | """Return how many shifts we have""" 84 | return len(self._shifts) 85 | 86 | def get_first_time_demand_not_met(self): 87 | """Find the first time the demand is not met""" 88 | for t in range(self.demand_length): 89 | if self._demand[t] > self._coverage[t]: 90 | return t 91 | 92 | raise Exception("Infeasible answer- demand is met %s %s %s" % 93 | (self.demand_is_met, self._demand, self._coverage)) 94 | 95 | @property 96 | def is_optimal(self): 97 | for t in range(self.demand_length): 98 | delta = self.get_demand_minus_coverage(t) 99 | if delta != 0: 100 | return False 101 | 102 | if delta < 0: 103 | return False 104 | 105 | return True 106 | 107 | def anneal(self): 108 | """Look for overages and try to fix them""" 109 | if not self.demand_is_met: 110 | raise Exception("Cannot anneal an unfeasible demand") 111 | 112 | if self.is_optimal: 113 | # noop 114 | return 115 | 116 | # Run on loop until no more improvements 117 | improvement_made = True 118 | time_saved = 0 119 | while improvement_made: 120 | improvement_made = False 121 | 122 | # Find times that are overscheduled 123 | for t in range(self.demand_length): 124 | if self.get_demand_minus_coverage(t) < 0: 125 | # Then it's unoptimal - look for shifts that start or end there, and try to roll back 126 | for i in range(len(self._shifts)): 127 | start, length = self._shifts[i] 128 | end = start + length 129 | if start == t and length > self.min_length: 130 | self._shifts[i] = (start + 1, length - 1) 131 | self._coverage[t] -= 1 132 | time_saved += 1 133 | improvement_made = True 134 | 135 | if end == t and length > self.max_length: 136 | self._shifts[i] = (start, length - 1) 137 | self._coverage[t] -= 1 138 | time_saved += 1 139 | improvement_made = True 140 | 141 | if time_saved > 0: 142 | logger.info("Annealing removed %s units", time_saved) 143 | 144 | if self.is_optimal: 145 | logger.info("Annealing reached optimality") 146 | -------------------------------------------------------------------------------- /chomp/splitter.py: -------------------------------------------------------------------------------- 1 | import copy 2 | 3 | from chomp import logger 4 | from chomp.decompose import Decompose 5 | from chomp.exceptions import UnequalDayLengthException 6 | 7 | 8 | class Splitter(object): 9 | """Take demand and turn it into subproblems for decompose.py 10 | 11 | This file is unitless and hypothetically support weeks of any length 12 | or days of any consistent length (e.g. 30 minute granularity) 13 | 14 | Demand is flattened then split into subproblems. The demand 15 | is considered circular such that, if a business is open from 8pm to 2am, 16 | then the final day of the week will wrap into the following week. 17 | 18 | 24/7 demand is split day by day right now. In the future, splitting it into 19 | overlapping sections (e.g. a midnight->noon, 8am-4pm, and noon-midnight) 20 | may be best for handling large demand. 21 | """ 22 | 23 | def __init__(self, week_demand, min_length, max_length): 24 | """Flatten demand and build helpers""" 25 | # week_demand is a list of lists 26 | 27 | # These are treated as unitless and should match the demand units 28 | logger.debug("Min %s max %s", min_length, max_length) 29 | self.min_length = int(min_length) 30 | self.max_length = int(max_length) 31 | 32 | self._shifts = [] # don't access directly! 33 | self._windows = [] 34 | 35 | self.week_length = len(week_demand) 36 | 37 | # Validate that days are same length 38 | self.day_length = len(week_demand[0]) 39 | for i in range(len(week_demand)): 40 | if self.day_length != len(week_demand[i]): 41 | raise UnequalDayLengthException() 42 | 43 | # 2) Flatten demand 44 | self.flat_demand = [ 45 | item for day_demand in week_demand for item in day_demand 46 | ] 47 | 48 | def calculate(self): 49 | # Generate subproblems 50 | self._generate_windows() 51 | self._solve_windows() 52 | 53 | def get_shifts(self): 54 | # Remove window and return shifts day by day 55 | shifts = copy.copy(self._shifts) 56 | for shift in shifts: 57 | shift["day"] = self._flat_index_to_day(shift["start"]) 58 | shift["start"] = self._flat_index_to_time(shift["start"]) 59 | 60 | return shifts 61 | 62 | def _generate_windows(self): 63 | """Generate the demand subproblems to solve.""" 64 | # Inclusive -> Exclusive (python list syntax) 65 | 66 | # Check for 24/7 edge case 67 | if self._is_always_open(): 68 | # Break it into day by day 69 | for i in range(self.week_length): 70 | start = i * self.day_length 71 | end = start + self.day_length 72 | self._add_window(start, end) 73 | 74 | return 75 | 76 | # Stop condition is based on start becuase of circular wraping 77 | for start in range(len(self.flat_demand)): 78 | if (self.flat_demand[start] is not 0) and ( 79 | start is 0 or self.flat_demand[start - 1] is 0): 80 | for end in range(start + 1, 81 | len(self.flat_demand) + self.max_length): 82 | 83 | if self._get_flat_demand(end) is 0 and ( 84 | start == (end - 1) or 85 | self._get_flat_demand(end - 1) is not 0): 86 | # Add to window . . . mayb 87 | # off by 1 becuase of exclusive 88 | self._add_window(start, end) 89 | break 90 | 91 | def _get_flat_demand(self, index): 92 | """Get flat demand at index - noting that it may be circular!""" 93 | return self.flat_demand[(index + 1) % len(self.flat_demand) - 1] 94 | 95 | def _add_window(self, start, end, raise_on_min_length=False): 96 | """Add a window - checking whether it violates rules""" 97 | 98 | length = end - start 99 | 100 | if length < self.min_length: 101 | # Only raise when recursing 102 | if raise_on_min_length: 103 | raise Exception("Min length constraint violated") 104 | 105 | if start == 0: 106 | # Expected 107 | logger.debug( 108 | "Skipping circular wraparound at beginning of loop") 109 | else: 110 | # Bad user. Bad. 111 | logger.info("Skipping window less than min length") 112 | return 113 | if length > self.day_length: 114 | # Split in two - a floor and a ceiling 115 | # Hypothetically recurses 116 | logger.info("Splitting large window into subproblems") 117 | center = start + (end - start) / 2 118 | # It's possible the windows will violate min length constrainnts, 119 | # so we wrap in a try block 120 | try: 121 | self._add_window(start, center, raise_on_min_length=True) 122 | self._add_window(center, end, raise_on_min_length=True) 123 | except: 124 | self._windows.append((start, end)) 125 | return 126 | 127 | self._windows.append((start, end)) 128 | 129 | def _solve_windows(self): 130 | """Run windows through decompose to create shifts""" 131 | 132 | window_count = 0 133 | for (start, stop) in self._windows: 134 | window_count += 1 135 | logger.info("Starting window %s of %s (start %s stop %s) ", 136 | window_count, len(self._windows), start, stop) 137 | 138 | # Need to wrap 139 | demand = self._get_window_demand(start, stop) 140 | d = Decompose( 141 | demand, self.min_length, self.max_length, window_offset=start) 142 | d.calculate() 143 | e = d.efficiency() 144 | logger.info("Window efficiency: Overage is %s percent", 145 | (e * 100.0)) 146 | self._shifts.extend(d.get_shifts()) 147 | 148 | # 149 | # helper methods 150 | # 151 | 152 | def _is_always_open(self): 153 | """Detect whether the business is 24/7""" 154 | # Hacky fuzzing due to a client that MOSTLY 24/7 155 | i = -1 156 | for d in self.flat_demand: 157 | i += 1 158 | # Equal because max length is not possible 159 | if d is 0 and i >= self.max_length: 160 | return False 161 | return True 162 | 163 | def _flat_index_to_day(self, index): 164 | """Take the flat demand index and return the day integer""" 165 | if index is 0: 166 | return 0 167 | 168 | # Python 2.7 integer division returns integer (but not true in py3) 169 | return index / self.day_length 170 | 171 | def _flat_index_to_time(self, index): 172 | """Take the flat demand index and return the start time integer""" 173 | return index % self.day_length 174 | 175 | def _is_circular_necessary(self): 176 | """Return whether the demand must be calculated circularly""" 177 | # Think of a business open until 2am every day with min shift length 3 178 | # Loading demand at the beginning of the week will show 2 hours 179 | # orphaned. It should be pushed onto the last day's subproblem. 180 | 181 | # Find first zero index 182 | first_zero = -1 183 | for t in range(len(self.flat_demand)): 184 | if self.flat_demand[t] is 0: 185 | first_zero = t 186 | break 187 | 188 | return first_zero < self.min_length 189 | 190 | # 191 | # Validation functions - mainly for testing 192 | # 193 | 194 | def validate(self): 195 | """Check whether shifts meet demand. Used in testing.""" 196 | expected_demand = copy.copy(self.flat_demand) 197 | sum_demand = [0] * len(self.flat_demand) 198 | 199 | logger.debug("Starting validation of %s shifts", len(self._shifts)) 200 | 201 | for shift in self._shifts: 202 | # Inclusive range 203 | for t in range(shift["start"], shift["start"] + shift["length"]): 204 | # (circular) 205 | sum_demand[(t + 1) % len(self.flat_demand) - 1] += 1 206 | 207 | logger.debug("Expected demand: %s", expected_demand) 208 | logger.debug("Scheduled supply: %s", sum_demand) 209 | for t in range(len(expected_demand)): 210 | if sum_demand[t] < expected_demand[t]: 211 | logger.error( 212 | "Demand not met at time %s (demand %s, supply %s)", t, 213 | expected_demand[t], sum_demand[t]) 214 | raise Exception("Demand not met at time %s" % t) 215 | return True 216 | 217 | def efficiency(self): 218 | """Return the overage as a float. 0 is perfect.""" 219 | PERFECT_OPTIMALITY = 0.0 220 | 221 | # Check for divide by 0 error. 222 | if sum(self.flat_demand) == 0.0: 223 | return PERFECT_OPTIMALITY 224 | 225 | efficiency = (1.0 * sum(shift["length"] for shift in self._shifts) / 226 | sum(self.flat_demand)) - 1 227 | 228 | logger.info("Efficiency: Overage is %s percent", efficiency * 100.0) 229 | return efficiency 230 | 231 | def _get_window_demand(self, start, stop): 232 | """Get circular start and stop (stop may wrap)""" 233 | if stop < len(self.flat_demand): 234 | return self.flat_demand[start:stop] 235 | return self.flat_demand[start:] + self.flat_demand[:stop % len( 236 | self.flat_demand)] 237 | -------------------------------------------------------------------------------- /chomp/tasking.py: -------------------------------------------------------------------------------- 1 | from time import sleep 2 | from copy import deepcopy 3 | from datetime import timedelta 4 | import traceback 5 | import os 6 | 7 | import pytz 8 | import iso8601 9 | from staffjoy import Client, NotFoundException 10 | 11 | from chomp.helpers import week_day_range, normalize_to_midnight 12 | from chomp import config, logger, Splitter 13 | 14 | 15 | class Tasking(): 16 | """Get tasks and process them""" 17 | 18 | REQUEUE_STATE = "chomp-queue" 19 | 20 | def __init__(self): 21 | self.client = Client(key=config.STAFFJOY_API_KEY, env=config.ENV) 22 | self.default_tz = pytz.timezone(config.DEFAULT_TZ) 23 | 24 | # To be defined later 25 | self.org = None 26 | self.loc = None 27 | self.role = None 28 | self.sched = None 29 | self.demand = None 30 | 31 | def server(self): 32 | previous_request_failed = False # Have some built-in retries 33 | 34 | while True: 35 | # Get task 36 | try: 37 | task = self.client.claim_chomp_task() 38 | logger.info("Task received: %s", task.data) 39 | previous_request_failed = False 40 | except NotFoundException: 41 | logger.debug("No task found. Sleeping.") 42 | previous_request_failed = False 43 | sleep(config.TASKING_FETCH_INTERVAL_SECONDS) 44 | continue 45 | except Exception as e: 46 | if not previous_request_failed: 47 | # retry, but info log it 48 | logger.info("Unable to fetch chomp task - retrying") 49 | previous_request_failed = True 50 | else: 51 | logger.error( 52 | "Unable to fetch chomp task after previous failure: %s", 53 | e) 54 | 55 | # Still sleep so we avoid thundering herd 56 | sleep(config.TASKING_FETCH_INTERVAL_SECONDS) 57 | continue 58 | 59 | try: 60 | self._process_task(task) 61 | task.delete() 62 | logger.info("Task completed %s", task.data) 63 | except Exception as e: 64 | logger.error("Failed schedule %s: %s %s", 65 | task.data.get("schedule_id"), e, 66 | traceback.format_exc()) 67 | 68 | logger.info("Requeuing schedule %s", 69 | task.data.get("schedule_id")) 70 | # self.sched set in process_task 71 | self.sched.patch(state=self.REQUEUE_STATE) 72 | 73 | # Sometimes rebooting Chomp helps with errors. For example, if 74 | # a Gurobi connection is drained then it helps to reboot. 75 | if config.KILL_ON_ERROR: 76 | sleep(config.KILL_DELAY) 77 | logger.info("Rebooting to kill container") 78 | os.system("shutdown -r now") 79 | 80 | def _process_task(self, task): 81 | # 1. Fetch schedule 82 | self.org = self.client.get_organization( 83 | task.data.get("organization_id")) 84 | self.loc = self.org.get_location(task.data.get("location_id")) 85 | self.role = self.loc.get_role(task.data.get("role_id")) 86 | self.sched = self.role.get_schedule(task.data.get("schedule_id")) 87 | 88 | self._compute_demand() 89 | self._subtract_existing_shifts_from_demand() 90 | 91 | # Run the calculation 92 | s = Splitter(self.demand, 93 | self.sched.data.get("min_shift_length_hour"), 94 | self.sched.data.get("max_shift_length_hour")) 95 | s.calculate() 96 | s.efficiency() 97 | 98 | # Naive becuase not yet datetimes 99 | naive_shifts = s.get_shifts() 100 | logger.info("Starting upload of %s shifts", len(naive_shifts)) 101 | 102 | local_start_time = self._get_local_start_time() 103 | 104 | for shift in naive_shifts: 105 | # We have to think of daylight savings time here, so we need to 106 | # guarantee that we don't have any errors. We do this by overshooting 107 | # the timedelta by an extra two hours, then rounding back to midnight. 108 | 109 | logger.debug("Processing shift %s", shift) 110 | 111 | start_day = normalize_to_midnight( 112 | deepcopy(local_start_time) + timedelta(days=shift["day"])) 113 | 114 | # Beware of time changes - duplicate times are possible 115 | try: 116 | start = start_day.replace(hour=shift["start"]) 117 | except pytz.AmbiguousTimeError: 118 | # Randomly pick one. Minor tech debt. 119 | start = start_day.replace(hour=shift["start"], is_dst=False) 120 | 121 | stop = start + timedelta(hours=shift["length"]) 122 | 123 | # Convert to the strings we are passing up to the cLoUd 124 | utc_start_str = start.astimezone(self.default_tz).isoformat() 125 | utc_stop_str = stop.astimezone(self.default_tz).isoformat() 126 | 127 | logger.info("Creating shift with start %s stop %s", start, stop) 128 | self.role.create_shift(start=utc_start_str, stop=utc_stop_str) 129 | 130 | def _subtract_existing_shifts_from_demand(self): 131 | logger.info("Starting demand: %s", self.demand) 132 | demand_copy = deepcopy(self.demand) 133 | search_start = (self._get_local_start_time() - timedelta( 134 | hours=config.MAX_SHIFT_LENGTH_HOURS)).astimezone(self.default_tz) 135 | # 1 week 136 | search_end = (self._get_local_start_time() + timedelta( 137 | days=7, hours=config.MAX_SHIFT_LENGTH_HOURS) 138 | ).astimezone(self.default_tz) 139 | 140 | shifts = self.role.get_shifts(start=search_start, end=search_end) 141 | 142 | logger.info("Checking %s shifts for existing demand", len(shifts)) 143 | 144 | # Search hour by hour throughout the weeks 145 | for day in range(len(self.demand)): 146 | start_day = normalize_to_midnight(self._get_local_start_time() + 147 | timedelta(days=day)) 148 | for start in range(len(self.demand[0])): 149 | 150 | # Beware of time changes - duplicate times are possible 151 | try: 152 | start_hour = deepcopy(start_day).replace(hour=start) 153 | except pytz.AmbiguousTimeError: 154 | # Randomly pick one - cause phucket. Welcome to chomp. 155 | start_hour = deepcopy(start_day).replace( 156 | hour=start, is_dst=False) 157 | 158 | try: 159 | stop_hour = start_hour + timedelta(hours=1) 160 | except pytz.AmbiguousTimeError: 161 | stop_hour = start_hour + timedelta(hours=1, is_dst=False) 162 | 163 | # Find shift 164 | current_staffing_level = 0 165 | for shift in shifts: 166 | shift_start = iso8601.parse_date( 167 | shift.data.get("start")).replace( 168 | tzinfo=self.default_tz) 169 | shift_stop = iso8601.parse_date( 170 | shift.data.get("stop")).replace(tzinfo=self.default_tz) 171 | 172 | if ((shift_start <= start_hour and shift_stop > stop_hour) 173 | or 174 | (shift_start >= start_hour and shift_start < stop_hour) 175 | or 176 | (shift_stop > start_hour and shift_stop <= stop_hour)): 177 | 178 | # increment staffing level during that bucket 179 | current_staffing_level += 1 180 | 181 | logger.debug("Current staffing level at day %s time %s is %s", 182 | day, start, current_staffing_level) 183 | 184 | demand_copy[day][start] -= current_staffing_level 185 | # demand cannot be less than zero 186 | if demand_copy[day][start] < 0: 187 | demand_copy[day][start] = 0 188 | 189 | logger.info("Demand minus existing shifts: %s", demand_copy) 190 | self.demand = demand_copy 191 | 192 | def _get_local_start_time(self): 193 | # Create the datetimes 194 | local_tz = pytz.timezone(self.loc.data.get("timezone")) 195 | utc_start_time = iso8601.parse_date( 196 | self.sched.data.get("start")).replace(tzinfo=self.default_tz) 197 | local_start_time = utc_start_time.astimezone(local_tz) 198 | return local_start_time 199 | 200 | def _compute_demand(self): 201 | weekday_demand = self.sched.data.get("demand") 202 | day_week_starts = self.org.data.get("day_week_starts") 203 | # flatten days from dict to list 204 | demand = [] 205 | for day in week_day_range(day_week_starts): 206 | demand.append(weekday_demand[day]) 207 | 208 | self.demand = demand 209 | -------------------------------------------------------------------------------- /conf/nginx-app.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80 default_server; 3 | server_name chomp-dev.staffjoy.com chomp-stage.staffjoy.com chomp-prod.staffjoy.com 4 | charset utf-8; 5 | error_log /dev/stdout crit; 6 | client_max_body_size 10M; 7 | location / { 8 | return 200 "chomp online"; 9 | add_header Content-Type text/plain; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /conf/supervisor-app.conf: -------------------------------------------------------------------------------- 1 | [program:nginx-app] 2 | command = /usr/sbin/nginx 3 | 4 | [program:memcached] 5 | command=/usr/bin/memcached -m 64 -p 11211 -u memcache -l 127.0.0.1 -DFOREGROUND 6 | 7 | [program:python-app] 8 | command= /src/server.sh 9 | -------------------------------------------------------------------------------- /functional-tests/__init__.py: -------------------------------------------------------------------------------- 1 | OPTIMALITY_THRESHHOLD = .1 # 10% 2 | -------------------------------------------------------------------------------- /functional-tests/test_decompose.py: -------------------------------------------------------------------------------- 1 | from chomp import Decompose, cache 2 | import pytest 3 | 4 | EFFICIENCY_LIMIT = .8 5 | 6 | 7 | class TestDecompose(): 8 | def setup_method(self, method): 9 | cache.flush() 10 | 11 | def teardown_method(self, method): 12 | cache.flush() 13 | 14 | def test_ondemand_data(self): 15 | """Data from customer bike group, Jan 2015""" 16 | demand = [ 17 | 0, 0, 0, 0, 0, 0, 0, 5, 5, 7, 8, 6, 6, 7, 7, 7, 9, 9, 6, 5, 4, 4, 18 | 0, 0 19 | ] 20 | min_length = 4 21 | max_length = 8 22 | d = Decompose(demand, min_length, max_length) 23 | d.calculate() 24 | d.validate() 25 | 26 | efficiency = d.efficiency() 27 | assert efficiency < EFFICIENCY_LIMIT 28 | 29 | # 10 min timeout (because it should hit subproblems) 30 | @pytest.mark.timeout(600) 31 | def test_big_on_demand(self): 32 | """Data from on demand client Jan 2015 (before splitting into roles)""" 33 | demand = [ 34 | 0, 0, 0, 0, 0, 0, 35, 35, 35, 34, 56, 59, 63, 70, 87, 107, 90, 61, 35 | 44, 32, 28 36 | ] 37 | min_length = 4 38 | max_length = 8 39 | d = Decompose(demand, min_length, max_length) 40 | d.calculate() 41 | d.validate() 42 | 43 | efficiency = d.efficiency() 44 | assert efficiency < EFFICIENCY_LIMIT 45 | 46 | @pytest.mark.timeout(600) 47 | def test_doesnt_freak_out_with_interior_zero(self): 48 | """Add an interior zero and make sure the algorithm does not freak out.""" 49 | # This is important becuase when bifurcating, an interior zero is possible 50 | # Here, it's infeasible unless the interior point is > 0 51 | demand = [1, 0, 1, 1, 2, 2, 2, 2, 3, 2, 2, 1] 52 | min_length = 2 53 | max_length = 4 54 | 55 | d = Decompose(demand, min_length, max_length) 56 | d.calculate() 57 | d.validate() 58 | 59 | efficiency = d.efficiency() 60 | assert efficiency < EFFICIENCY_LIMIT 61 | -------------------------------------------------------------------------------- /functional-tests/test_splitter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from chomp import Splitter, cache 4 | 5 | EFFICIENCY_LIMIT = .8 6 | PERFECT_OPTIMALITY = 0.0 7 | 8 | 9 | class TestSplitter(): 10 | def setup_method(self, method): 11 | cache.flush() 12 | 13 | def teardown_method(self, method): 14 | cache.flush() 15 | 16 | def test_succeeds_with_zero_demand(self): 17 | # yapf: disable 18 | demand = [ 19 | [0]*24, 20 | [0]*24, 21 | [0]*24, 22 | [0]*24, 23 | [0]*24, 24 | [0]*24, 25 | [0]*24, 26 | ] 27 | # yapf: enable 28 | 29 | min_length = 4 30 | max_length = 8 31 | 32 | s = Splitter(demand, min_length, max_length) 33 | s.calculate() 34 | s.validate() 35 | 36 | efficiency = s.efficiency() 37 | assert efficiency == PERFECT_OPTIMALITY 38 | 39 | @pytest.mark.timeout(1600) 40 | def test_on_demand_old(self): 41 | """Old on demand data - good test of wrap-around""" 42 | # yapf: disable 43 | demand = [ 44 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 3, 4, 4, 6, 6, 7, 7, 5, 2], 45 | [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 4, 5, 3, 3, 4, 4, 7, 6, 5, 4, 2], 46 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 5, 4, 5, 4, 5, 5, 7, 6, 6, 5, 3], 47 | [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 3, 5, 5, 2, 5, 6, 7, 6, 6, 5, 3], 48 | [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 4, 3, 5, 4, 6, 5, 6, 8, 5, 4, 4], 49 | [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 5, 3, 4, 5, 6, 7, 8, 6, 6, 5, 2], 50 | [2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 5, 5, 5, 4, 6, 6, 11, 8, 6, 4, 2], 51 | ] 52 | # yapf: enable 53 | 54 | min_length = 4 55 | max_length = 8 56 | 57 | s = Splitter(demand, min_length, max_length) 58 | s.calculate() 59 | s.validate() 60 | 61 | efficiency = s.efficiency() 62 | assert efficiency < EFFICIENCY_LIMIT 63 | 64 | @pytest.mark.timeout(1600) 65 | def test_la(self): 66 | """24/7 LA client data""" 67 | # yapf: disable 68 | demand = [ 69 | # They sometimes have leading zeros 70 | [0, 0, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 1], 71 | [1, 1, 1, 1, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3], 72 | [4, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4], 73 | [3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3], 74 | [3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4], 75 | [4, 3, 3, 3, 3, 5, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], 76 | [4, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4], 77 | ] 78 | # yapf: enable 79 | min_length = 6 80 | max_length = 8 81 | 82 | s = Splitter(demand, min_length, max_length) 83 | s.calculate() 84 | s.validate() 85 | assert s.efficiency() < 0.08 86 | 87 | @pytest.mark.timeout(1600) 88 | def test_call_center(self): 89 | """See if it chokes for call center client data""" 90 | # yapf: disable 91 | demand = [ 92 | [0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 0, 0, 0, 0, 0], 93 | [0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 0, 0, 0, 0, 0], 94 | [0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 0, 0, 0, 0, 0], 95 | [0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 0, 0, 0, 0, 0], 96 | [0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 0, 0, 0, 0, 0], 97 | [0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 0, 0, 0, 0, 0], 98 | [0, 0, 0, 0, 0, 0, 0, 0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 0, 0, 0, 0, 0], 99 | ] 100 | # yapf: enable 101 | min_length = 9 102 | max_length = 9 103 | 104 | s = Splitter(demand, min_length, max_length) 105 | s.calculate() 106 | s.validate() 107 | # For these, we really need perfect efficiency 108 | assert s.efficiency() < 0.001 109 | 110 | @pytest.mark.timeout(1600) 111 | def test_london_on_demand_two(self): 112 | """Test on demand trial data that supposedly had low efficiency""" 113 | # yapf: disable 114 | demand = [ 115 | [0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 4, 8, 12, 10, 9, 8, 9, 15, 15, 20, 116 | 17, 14, 7, 4], 117 | [0, 0, 0, 0, 0, 0, 0, 0, 3, 4, 6, 10, 14, 14, 9, 9, 9, 12, 20, 25, 118 | 20, 12, 9, 5], 119 | [0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 7, 12, 15, 12, 10, 9, 12, 12, 16, 120 | 21, 20, 12, 9, 4], 121 | [0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 7, 10, 13, 13, 9, 9, 11, 15, 17, 24, 122 | 23, 14, 7, 5], 123 | [0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 7, 10, 13, 13, 11, 11, 11, 12, 25, 124 | 29, 29, 20, 12, 9], 125 | [6, 0, 0, 0, 0, 0, 0, 0, 3, 6, 6, 12, 19, 18, 17, 16, 16, 18, 24, 126 | 25, 25, 18, 11, 7], 127 | [4, 0, 0, 0, 0, 0, 0, 0, 3, 6, 7, 12, 19, 19, 17, 16, 16, 18, 23, 128 | 29, 26, 14, 9, 5], 129 | ] 130 | # yapf: enable 131 | min_length = 4 132 | max_length = 14 133 | 134 | s = Splitter(demand, min_length, max_length) 135 | s.calculate() 136 | s.validate() 137 | # For these, we really need perfect efficiency 138 | assert s.efficiency() < 0.01 139 | 140 | def test_online_store_prod_failure(self): 141 | """This data caused a production error""" 142 | # yapf: disable 143 | demand = [ 144 | [8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8], 145 | [8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8], 146 | [8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8], 147 | [8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 148 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 149 | [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8], 150 | [8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8], 151 | ] 152 | # yapf: enable 153 | min_length = 8 154 | max_length = 8 155 | 156 | s = Splitter(demand, min_length, max_length) 157 | s.calculate() 158 | s.validate() 159 | 160 | # We commented out this test because it's making Travis-ci.org time out. 161 | # Yes, it's ironic that the slow test causes failure because it's slow, but in our internal 162 | # tests the processors were fast enough in CI and prod for an acceptable run time of this 163 | # problem :-) 164 | """ 165 | def test_labs_slow(self): 166 | # yapf: disable 167 | demand = [ 168 | [0, 0, 0, 0, 1, 1, 4, 4, 2, 5, 5, 5, 5, 5, 5, 4, 4, 4, 4, 2, 2, 2, 0, 0], 169 | [0, 0, 0, 0, 2, 6, 10, 12, 14, 12, 17, 15, 16, 19, 14, 11, 9, 10, 9, 5, 5, 4, 2, 2], 170 | [1, 2, 1, 2, 2, 6, 9, 10, 11, 13, 12, 13, 14, 11, 12, 12, 9, 5, 4, 2, 2, 0, 0, 0], 171 | [2, 1, 1, 1, 2, 6, 8, 9, 11, 16, 16, 11, 16, 15, 11, 16, 11, 17, 7, 13, 7, 5, 5, 2], 172 | [1, 0, 1, 1, 2, 5, 9, 15, 17, 15, 13, 14, 13, 13, 13, 9, 10, 11, 6, 6, 6, 4, 1, 2], 173 | [1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 0, 0, 0, 0, 1, 4, 2, 1, 1, 1, 2, 2, 2], 174 | [0, 0, 0, 0, 1, 1, 1, 1, 1, 2, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0], 175 | ] 176 | # yapf: enable 177 | min_length = 4 178 | max_length = 8 179 | 180 | s = Splitter(demand, min_length, max_length) 181 | s.calculate() 182 | s.validate() 183 | """ -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | 3 | unit-test: 4 | rm -rf tests/__pycache__/ 5 | py.test tests -v -s 6 | 7 | functional-test: 8 | rm -rf functional-tests/__pycache__/ 9 | py.test functional-tests -v -s 10 | test: 11 | make fmt-test 12 | make py-lint 13 | make unit-test 14 | make functional-test 15 | 16 | dependencies: 17 | pip install -I -r requirements.txt 18 | pip freeze > requirements.txt 19 | 20 | fmt: 21 | yapf -r -i chomp/ functional-tests/ tests/ || : 22 | 23 | fmt-test: 24 | yapf -r -d chomp/ functional-tests/ tests/ || (echo "Document not formatted - run 'make fmt'" && exit 1) 25 | dev-test: 26 | make fmt 27 | make test 28 | dependency-update: 29 | echo "Updating all packages in requirements.txt" 30 | pip freeze --local | grep -v '^\-e' | cut -d = -f 1 | xargs -n1 pip install -U 31 | pip freeze > requirements.txt 32 | server: 33 | bash server.sh 34 | py-lint: 35 | pylint --rcfile .pylintrc chomp/ 36 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.2 2 | astroid==1.4.5 3 | cffi==1.6.0 4 | colorama==0.3.7 5 | cryptography==1.3.2 6 | enum34==1.1.6 7 | idna==2.1 8 | ipaddress==1.0.16 9 | iso8601==0.1.11 10 | lazy-object-proxy==1.2.2 11 | ndg-httpsclient==0.4.0 12 | packaging==16.8 13 | py==1.4.31 14 | pyasn1==0.1.9 15 | pycparser==2.14 16 | pylint==1.5.5 17 | pyOpenSSL==16.0.0 18 | pyparsing==2.1.10 19 | pytest==2.9.1 20 | python-memcached==1.58 21 | pytz==2016.4 22 | requests==2.10.0 23 | six==1.10.0 24 | staffjoy==0.24 25 | wrapt==1.10.8 26 | yapf==0.16.0 27 | -------------------------------------------------------------------------------- /server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | python -c "from chomp import Tasking; t = Tasking(); t.server()" 5 | exit 1 6 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | version = "1.0" 4 | setup(name="chomp", 5 | packages=find_packages(), 6 | version=version, 7 | description="Staffjoy V1 Forecast to Shifts Decomposition Tool", 8 | author="Philip Thomas", 9 | author_email="philip@staffjoy.com", 10 | license="MIT", 11 | url="https://github.com/staffjoy/chomp-decomposition", 12 | download_url="https://github.com/StaffJoy/chomp-decomposition/archive/%s.tar.gz" % version, 13 | keywords=["staffjoy-api", "staffjoy", "staff joy", "chomp"], 14 | install_requires=["requests[security]"], ) 15 | 16 | 17 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Staffjoy/chomp-decomposition/871729063176679c07c4a4b91f39d0fbcf7b1737/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_cache.py: -------------------------------------------------------------------------------- 1 | from chomp import cache 2 | 3 | 4 | class TestCache(): 5 | def setup_method(self, method): 6 | self.cache = cache 7 | self.cache.flush() 8 | 9 | def teardown_method(self, method): 10 | self.cache.flush() 11 | 12 | def test_cache_caches(self): 13 | demand = [1, 2, 3, 2, 1] 14 | min_length = 1 15 | max_length = 2 16 | 17 | demo_shifts = [{"start": 1, "length": 2}, {"start": 3, "length": 4}] 18 | 19 | assert self.cache.get( 20 | demand=demand, min_length=min_length, 21 | max_length=max_length) is None 22 | 23 | self.cache.set( 24 | demand=demand, 25 | min_length=min_length, 26 | max_length=max_length, 27 | shifts=demo_shifts) 28 | 29 | assert self.cache.get( 30 | demand=demand, 31 | min_length=min_length, 32 | max_length=max_length, ) == demo_shifts 33 | 34 | # Fuzz it and make sure still none 35 | assert self.cache.get( 36 | demand=demand, 37 | min_length=min_length, 38 | max_length=(max_length + 1), ) is None 39 | 40 | assert self.cache.get( 41 | demand=demand, 42 | min_length=(min_length + 1), 43 | max_length=max_length, ) is None 44 | 45 | assert self.cache.get( 46 | demand=demand.append(1), 47 | min_length=min_length, 48 | max_length=max_length, ) is None 49 | -------------------------------------------------------------------------------- /tests/test_decompose.py: -------------------------------------------------------------------------------- 1 | from chomp import Decompose, cache 2 | 3 | 4 | class TestDecompose(): 5 | def setup_method(self, method): 6 | cache.flush() 7 | 8 | def teardown_method(self, method): 9 | cache.flush() 10 | 11 | def test_init_no_shifts(self): 12 | demand = [1, 2, 3, 2, 1] 13 | min_length = 1 14 | max_length = 2 15 | 16 | d = Decompose(demand, min_length, max_length) 17 | 18 | assert d.demand == demand 19 | assert d.min_length == min_length 20 | assert d.max_length == max_length 21 | assert d.window_offset == 0 # No windowing 22 | 23 | def test_lagging_zeros_windowing(self): 24 | demand = [1, 2, 3, 2, 1, 0, 0] 25 | expected_processed_demand = [1, 2, 3, 2, 1] 26 | min_length = 1 27 | max_length = 2 28 | 29 | d = Decompose(demand, min_length, max_length) 30 | 31 | assert d.demand == expected_processed_demand 32 | assert d.min_length == min_length 33 | assert d.max_length == max_length 34 | assert d.window_offset == 0 # No windowing 35 | 36 | def test_leading_zeros_windowing(self): 37 | demand = [0, 0, 0, 1, 2, 3, 2, 1] 38 | expected_processed_demand = [1, 2, 3, 2, 1] 39 | min_length = 1 40 | max_length = 2 41 | 42 | d = Decompose(demand, min_length, max_length) 43 | 44 | assert d.demand == expected_processed_demand 45 | assert d.min_length == min_length 46 | assert d.max_length == max_length 47 | assert d.window_offset == 3 # Windowing 48 | 49 | def test_combined_windowing(self): 50 | demand = [0, 0, 0, 0, 1, 0, 2, 3, 0, 2, 1, 0, 0] 51 | expected_processed_demand = [1, 0, 2, 3, 0, 2, 1] 52 | min_length = 1 53 | max_length = 2 54 | 55 | d = Decompose(demand, min_length, max_length) 56 | 57 | assert d.demand == expected_processed_demand 58 | assert d.window_offset == 4 # Windowing 59 | assert d.min_length == min_length 60 | assert d.max_length == max_length 61 | 62 | def test_subproblem_generation(self): 63 | demand = [0, 1, 2, 3, 4, 2] 64 | min_length = 1 65 | max_length = 2 66 | 67 | expected_window_offset = 1 68 | expected_windowed_demand = [1, 2, 3, 4, 2] 69 | expected_round_up = [1, 1, 2, 2, 1] 70 | expected_round_down = [0, 1, 1, 2, 1] 71 | 72 | d = Decompose(demand, min_length, max_length) 73 | 74 | actual_round_up = d._split_demand(round_up=True) 75 | actual_round_down = d._split_demand(round_up=False) 76 | actual_window_offset = d.window_offset 77 | 78 | assert actual_round_up == expected_round_up 79 | assert actual_round_down == expected_round_down 80 | assert actual_window_offset == expected_window_offset 81 | 82 | # Check that we exactly split demand in 2 (no overage or underage) 83 | recombined = [ 84 | up + down for up, down in zip(actual_round_up, actual_round_down) 85 | ] 86 | assert recombined == expected_windowed_demand 87 | 88 | def test_edge_smoothing(self): 89 | demand = [3, 3, 2, 2, 4, 2, 3, 1, 3] 90 | min_length = 3 91 | max_length = 4 92 | expected_demand = [3, 3, 3, 2, 4, 3, 3, 3, 3] 93 | d = Decompose(demand, min_length, max_length) 94 | assert d.demand == expected_demand 95 | -------------------------------------------------------------------------------- /tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | import pytest 4 | import pytz 5 | 6 | from chomp.helpers import week_day_range, normalize_to_midnight, inclusive_range, reverse_inclusive_range 7 | 8 | 9 | def test_week_day_range_throws_error_for_invalid_day(): 10 | with pytest.raises(ValueError): 11 | week_day_range("miercoles") 12 | 13 | 14 | def test_week_day_range_succeeds_with_no_input(): 15 | week_day_range() 16 | 17 | 18 | def test_days_of_week_returns_correct_days_of_week_order(): 19 | expected = [ 20 | "wednesday", 21 | "thursday", 22 | "friday", 23 | "saturday", 24 | "sunday", 25 | "monday", 26 | "tuesday", 27 | ] 28 | actual = week_day_range(expected[0]) 29 | assert actual == expected 30 | 31 | 32 | def test_normalize_to_midnight(): 33 | start = datetime( 34 | 1990, 12, 9, 6, 22, 11, tzinfo=pytz.timezone("US/Eastern")) 35 | expected = datetime( 36 | 1990, 12, 9, 0, 0, 0, tzinfo=pytz.timezone("US/Eastern")) 37 | assert normalize_to_midnight(start) == expected 38 | 39 | 40 | def test_inclusive_range(): 41 | low = -1 42 | high = 1 43 | expected = [-1, 0, 1] 44 | assert inclusive_range(low, high) == expected 45 | 46 | 47 | def test_reverse_inclusive_range(): 48 | low = -1 49 | high = 1 50 | expected = [1, 0, -1] 51 | assert reverse_inclusive_range(low, high) == expected 52 | -------------------------------------------------------------------------------- /tests/test_shift_collection.py: -------------------------------------------------------------------------------- 1 | from chomp.shift_collection import ShiftCollection 2 | 3 | import pytest 4 | 5 | 6 | class TestShiftCollection(): 7 | def setup_method(self, method): 8 | self.min_length = 5 9 | self.max_length = 6 10 | self.demand = [1, 2, 3, 4, 5, 4, 3, 2, 1] 11 | self.demand_sum = sum(self.demand) 12 | self.collection = ShiftCollection( 13 | self.min_length, self.max_length, demand=self.demand) 14 | 15 | def teardown_method(self, method): 16 | pass 17 | 18 | def test_init_noshifts(self): 19 | assert self.collection.shifts == [] 20 | 21 | for t in range(len(self.demand)): 22 | assert self.collection.get_demand_minus_coverage(t) == self.demand[ 23 | t] 24 | 25 | assert self.collection.coverage_sum == 0 26 | assert len(self.demand) == self.collection.demand_length 27 | assert self.collection.best_possible_coverage == self.demand_sum 28 | assert self.collection.demand_is_met == False 29 | assert self.collection.shift_count == 0 30 | assert self.collection.get_first_time_demand_not_met() == 0 31 | assert self.collection.is_optimal == False 32 | 33 | def test_add_a_shift(self): 34 | # We intentionally add it from second time to end time 35 | start = 1 36 | length = 8 37 | shift = (start, length) 38 | expected_demand_minus_coverage = [1, 1, 2, 3, 4, 3, 2, 1, 0] 39 | 40 | self.collection.add_shift(shift) 41 | 42 | assert self.collection.shifts == [shift] 43 | 44 | for t in range(len(self.demand)): 45 | assert self.collection.get_demand_minus_coverage( 46 | t) == expected_demand_minus_coverage[t] 47 | 48 | assert self.collection.coverage_sum == length 49 | 50 | # No overage 51 | assert self.collection.best_possible_coverage == self.demand_sum 52 | assert self.collection.demand_is_met == False 53 | assert self.collection.shift_count == 1 54 | assert self.collection.get_first_time_demand_not_met() == 0 55 | assert self.collection.is_optimal == False 56 | 57 | def test_add_three_shifts(self): 58 | 59 | # Remember that they are (start, length) 60 | shifts = [(0, 3), (0, 3), (3, 4)] 61 | expected_demand_minus_coverage = [-1, 0, 1, 3, 4, 3, 2, 2, 1] 62 | 63 | for shift in shifts: 64 | self.collection.add_shift(shift) 65 | 66 | assert self.collection.shifts == shifts 67 | 68 | for t in range(len(self.demand)): 69 | assert self.collection.get_demand_minus_coverage( 70 | t) == expected_demand_minus_coverage[t] 71 | 72 | assert self.collection.coverage_sum == 10 73 | 74 | # We have triggered 1 overage, so it shoudl be demand plus length 75 | assert self.collection.best_possible_coverage == self.demand_sum + 1 76 | assert self.collection.demand_is_met == False 77 | assert self.collection.shift_count == 3 78 | assert self.collection.get_first_time_demand_not_met() == 2 79 | assert self.collection.is_optimal == False 80 | 81 | def test_add_overage_shifts(self): 82 | 83 | # Remember that they are (start, length) 84 | shifts = [(0, 9)] * 5 85 | expected_demand_minus_coverage = [-4, -3, -2, -1, 0, -1, -2, -3, -4] 86 | 87 | for shift in shifts: 88 | self.collection.add_shift(shift) 89 | 90 | assert self.collection.shifts == shifts 91 | 92 | for t in range(len(self.demand)): 93 | assert self.collection.get_demand_minus_coverage( 94 | t) == expected_demand_minus_coverage[t] 95 | 96 | assert self.collection.coverage_sum == 9 * 5 97 | 98 | # We have triggered 20 overage, so it should be demand plus length 99 | assert self.collection.best_possible_coverage == self.demand_sum + 20 100 | assert self.collection.demand_is_met == True 101 | assert self.collection.shift_count == 5 102 | with pytest.raises(Exception): 103 | assert self.collection.get_first_time_demand_not_met() == 2 104 | assert self.collection.is_optimal == False 105 | 106 | def test_add_optimal_shifts(self): 107 | 108 | # Remember that they are (start, length) 109 | shifts = [(0, 5), (1, 5), (2, 5), (3, 5), (4, 5)] 110 | 111 | for shift in shifts: 112 | self.collection.add_shift(shift) 113 | 114 | assert self.collection.shifts == shifts 115 | 116 | for t in range(len(self.demand)): 117 | # Optimal 118 | assert self.collection.get_demand_minus_coverage(t) == 0 119 | 120 | assert self.collection.coverage_sum == self.demand_sum 121 | 122 | assert self.collection.best_possible_coverage == self.demand_sum 123 | 124 | assert self.collection.demand_is_met == True 125 | assert self.collection.shift_count == 5 126 | with pytest.raises(Exception): 127 | assert self.collection.get_first_time_demand_not_met() == 2 128 | assert self.collection.is_optimal == True 129 | 130 | # Now we anneal and asser that nothing's done 131 | self.collection.anneal() 132 | assert self.collection.shifts == shifts 133 | assert self.collection.demand_is_met == True 134 | assert self.collection.is_optimal == True 135 | 136 | def test_adding_shift_below_range_triggers_exception(self): 137 | too_short_shift = (-1, 3) 138 | with pytest.raises(Exception): 139 | self.collection.add_shift(too_short_shift) 140 | 141 | def test_adding_shift_above_bounds_triggers_exception(self): 142 | too_long_shift = (5, 5) 143 | with pytest.raises(Exception): 144 | self.collection.add_shift(too_long_shift) 145 | 146 | def test_must_be_feasible_for_annealing(self): 147 | with pytest.raises(Exception): 148 | self.collection.anneal() 149 | 150 | def test_annealing_succeeds(self): 151 | # Similar to optimal shifts 152 | shifts = [(0, 5), (1, 5), (1, 6), (3, 5), (4, 5)] 153 | 154 | expected_annealed_shifts = [(0, 5), (1, 5), (2, 5), (3, 5), (4, 5)] 155 | 156 | for shift in shifts: 157 | self.collection.add_shift(shift) 158 | 159 | self.collection.anneal() 160 | assert self.collection.shifts == expected_annealed_shifts 161 | 162 | def test_annealing_min_shift_length_noop(self): 163 | # Basically re-run last test but with longer min shift length 164 | 165 | demand = [1, 2, 3, 4, 4, 4, 3, 2, 1] 166 | self.collection = ShiftCollection(5, 5, demand=demand) 167 | shifts = [(0, 5), (1, 5), (2, 5), (3, 5), (4, 5)] 168 | 169 | for shift in shifts: 170 | self.collection.add_shift(shift) 171 | 172 | self.collection.anneal() 173 | assert self.collection._shifts == shifts 174 | -------------------------------------------------------------------------------- /tests/test_splitter.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from chomp import Splitter 4 | from chomp.exceptions import UnequalDayLengthException 5 | 6 | 7 | class TestDecompose(): 8 | def setup_method(self, method): 9 | self.week_demand = [[1, 2, 3, 0], [1, 3, 1, 0], [1, 1, 1, 0]] 10 | self.expected_flat_week_demand = [1, 2, 3, 0, 1, 3, 1, 0, 1, 1, 1, 11 | 0] # len 12 12 | self.min_length = 3 13 | self.max_length = 4 14 | 15 | def teardown_method(self, method): 16 | pass 17 | 18 | def test_splitter_init(self): 19 | s = Splitter(self.week_demand, self.min_length, self.max_length) 20 | 21 | assert len(s.flat_demand) == len(self.week_demand) * len( 22 | self.week_demand[0]) 23 | assert s.flat_demand == self.expected_flat_week_demand 24 | assert s.min_length == self.min_length 25 | assert s.max_length == self.max_length 26 | assert s.day_length == len(self.week_demand[0]) 27 | 28 | def test_circular_get_window_demand(self): 29 | s = Splitter(self.week_demand, self.min_length, self.max_length) 30 | # For circular - I'll do one manual for sanity and a couple programmatic 31 | assert s._get_window_demand(1, 32 | 4) == self.expected_flat_week_demand[1:4] 33 | assert s._get_window_demand(11, 13) == [0, 1] 34 | assert s._get_window_demand(11, 11) == [] 35 | 36 | def test_circular_get_demand(self): 37 | s = Splitter(self.week_demand, self.min_length, self.max_length) 38 | # For circular - I'll do one manual for sanity and a couple programmatic 39 | assert s._get_flat_demand(13) == self.expected_flat_week_demand[1] 40 | for t in range(len(self.expected_flat_week_demand)): 41 | assert s._get_flat_demand(t) == self.expected_flat_week_demand[t] 42 | assert s._get_flat_demand(t + len(self.expected_flat_week_demand) 43 | ) == self.expected_flat_week_demand[t] 44 | 45 | def test_splitter_init_different_day_lengths(self): 46 | self.week_demand[1].pop() 47 | 48 | with pytest.raises(UnequalDayLengthException): 49 | 50 | Splitter(self.week_demand, self.min_length, self.max_length) 51 | 52 | def test_windowing_standard(self): 53 | s = Splitter(self.week_demand, self.min_length, self.max_length) 54 | s._generate_windows() 55 | expected_windows = [(0, 3), (4, 7), (8, 11)] 56 | 57 | assert s._windows == expected_windows 58 | 59 | def test_windowing_standard_again(self): 60 | self.week_demand = [[0, 2, 3, 4, 0, 3, 1, 8], [1, 2, 3, 0, 2, 3, 1, 8]] 61 | s = Splitter(self.week_demand, self.min_length, self.max_length) 62 | s._generate_windows() 63 | expected_windows = [(1, 4), (5, 11), (12, 16)] 64 | assert s._windows == expected_windows 65 | 66 | def test_windowing_always_open(self): 67 | self.week_demand = [[1, 2, 3, 4], [1, 3, 1, 8], [1, 1, 1, 2]] 68 | s = Splitter(self.week_demand, self.min_length, self.max_length) 69 | s._generate_windows() 70 | expected_windows = [(0, 4), (4, 8), (8, 12)] 71 | assert s._windows == expected_windows 72 | 73 | def test_windowing_short_subproblem(self): 74 | self.week_demand = [[1, 2, 0, 0], [1, 3, 1, 0], [0, 1, 1, 0]] 75 | s = Splitter(self.week_demand, self.min_length, self.max_length) 76 | s._generate_windows() 77 | expected_windows = [ 78 | (4, 7), 79 | ] 80 | assert s._windows == expected_windows 81 | 82 | def test_windowing_circular(self): 83 | self.week_demand = [[1, 1, 0, 4], [1, 2, 1, 0], [0, 0, 1, 1]] 84 | s = Splitter(self.week_demand, self.min_length, self.max_length) 85 | s._generate_windows() 86 | expected_windows = [(3, 7), (10, 14)] 87 | assert s._windows == expected_windows 88 | 89 | def test_windowing_split_invalid_subproblem(self): 90 | # Window that is so large that it gets recursively split . . . 91 | # BUT one of the split problems is less the min length, so 92 | # it shoudl abandon the search 93 | self.week_demand = [[1, 1, 0, 4], [1, 2, 1, 0], [0, 1, 1, 1]] 94 | s = Splitter(self.week_demand, self.min_length, self.max_length) 95 | s._generate_windows() 96 | expected_windows = [(3, 7), (9, 14)] 97 | assert s._windows == expected_windows 98 | 99 | def test_windowing_split_valid_subproblem(self): 100 | # Window so large that it gets recursively split 101 | self.min_length = 2 # Update versus prior test 102 | self.week_demand = [[1, 0, 0, 4], [1, 2, 1, 0], [1, 1, 1, 1]] 103 | s = Splitter(self.week_demand, self.min_length, self.max_length) 104 | s._generate_windows() 105 | expected_windows = [(3, 7), (8, 10), (10, 13)] 106 | assert s._windows == expected_windows 107 | 108 | # helper methods 109 | def test_is_always_open_case_true(self): 110 | self.week_demand = [[1, 2, 3, 4], [1, 3, 1, 8], [1, 1, 1, 2]] 111 | s = Splitter(self.week_demand, self.min_length, self.max_length) 112 | assert s._is_always_open() is True 113 | 114 | def test_is_always_open_case_carryover_true(self): 115 | self.week_demand = [[0, 0, 0, 4], [1, 3, 1, 8], [1, 1, 1, 2]] 116 | s = Splitter(self.week_demand, self.min_length, self.max_length) 117 | assert s._is_always_open() is True 118 | 119 | def test_is_always_open_case_false(self): 120 | s = Splitter(self.week_demand, self.min_length, self.max_length) 121 | # Just going through this whole week, cause fuck it 122 | assert s._is_always_open() is False 123 | 124 | def test_flat_index_to_day(self): 125 | """Take the flat demand index and return the day integer""" 126 | s = Splitter(self.week_demand, self.min_length, self.max_length) 127 | flat_to_day = [ 128 | (0, 0), 129 | (1, 0), 130 | (2, 0), 131 | (3, 0), 132 | (4, 1), 133 | (5, 1), 134 | (6, 1), 135 | (7, 1), 136 | (8, 2), 137 | (9, 2), 138 | (10, 2), 139 | (11, 2), 140 | ] 141 | 142 | for (flat, day) in flat_to_day: 143 | assert s._flat_index_to_day(flat) == day 144 | 145 | def test_flat_index_to_time(self): 146 | s = Splitter(self.week_demand, self.min_length, self.max_length) 147 | flat_to_time = [ 148 | (0, 0), 149 | (1, 1), 150 | (2, 2), 151 | (3, 3), 152 | (4, 0), 153 | (5, 1), 154 | (6, 2), 155 | (7, 3), 156 | (8, 0), 157 | (9, 1), 158 | (10, 2), 159 | (11, 3), 160 | ] 161 | 162 | for (flat, time) in flat_to_time: 163 | assert s._flat_index_to_time(flat) == time 164 | 165 | def test_is_circular_necessary_case_false(self): 166 | s = Splitter(self.week_demand, self.min_length, self.max_length) 167 | assert s._is_circular_necessary() is False 168 | 169 | def test_is_circular_necessary_case_true(self): 170 | self.week_demand = [[1, 1, 0, 4], [1, 2, 1, 0], [1, 1, 1, 1]] 171 | s = Splitter(self.week_demand, self.min_length, self.max_length) 172 | assert s._is_circular_necessary() is True 173 | -------------------------------------------------------------------------------- /vagrant.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | cd /vagrant/build 5 | bash ubuntu-install.sh 6 | cd .. 7 | 8 | # Virtual Env 9 | sudo pip install virtualenv 10 | echo "source /vagrant/vagrant-venv/bin/activate" >> $HOME/.bashrc 11 | echo 'echo "export PYTHONPATH=\"/vagrant/\"" >> /etc/profile' | sudo sh 12 | 13 | rm -rf vagrant-venv && virtualenv vagrant-venv 14 | 15 | source /etc/profile 16 | source /vagrant/vagrant-venv/bin/activate 17 | cd /vagrant/ 18 | make dependencies --------------------------------------------------------------------------------