├── .gitignore
├── .pylintrc
├── LICENSE
├── MANIFEST.in
├── README.rst
├── bin
    ├── climageprocessor
    └── climageserver
├── climage
    ├── __init__.py
    ├── exif.py
    ├── processor.py
    └── server.py
├── coverage.sh
├── doc
    ├── _build
    │   └── .gitignore
    ├── climage.exif.rst
    ├── climage.processor.rst
    ├── climage.server.rst
    ├── conf.py
    └── index.rst
├── setup.cfg
├── setup.py
└── test
    ├── __init__.py
    ├── test.jpg
    ├── test_exif.jpg
    ├── test_processor.py
    └── test_server.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.pyc
 2 | *.so
 3 | .coverage*
 4 | build
 5 | coverage.html
 6 | dist
 7 | climage.egg-info
 8 | test_blob
 9 | test_image_bad
10 | 


--------------------------------------------------------------------------------
/.pylintrc:
--------------------------------------------------------------------------------
 1 | [Basic]
 2 | method-rgx=([a-z_][a-z0-9_]{1,30}$)|setUp|tearDown
 3 | no-docstring-rgx=(__.*__)|([Tt]est.*)
 4 | 
 5 | [Design]
 6 | max-args=10
 7 | max-attributes=20
 8 | min-public-methods=0
 9 | # Need a lot here because unittest classes have a lot
10 | max-public-methods=100
11 | 
12 | [Messages Control]
13 | # Run 'pylint --list-msgs | grep <id>' to see what each ID means.
14 | disable=R0921,W0142,I0011,R0922,W0703,W0603
15 | 
16 | [Variables]
17 | additional-builtins=_
18 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 | 
  2 |                                  Apache License
  3 |                            Version 2.0, January 2004
  4 |                         http://www.apache.org/licenses/
  5 | 
  6 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  7 | 
  8 |    1. Definitions.
  9 | 
 10 |       "License" shall mean the terms and conditions for use, reproduction,
 11 |       and distribution as defined by Sections 1 through 9 of this document.
 12 | 
 13 |       "Licensor" shall mean the copyright owner or entity authorized by
 14 |       the copyright owner that is granting the License.
 15 | 
 16 |       "Legal Entity" shall mean the union of the acting entity and all
 17 |       other entities that control, are controlled by, or are under common
 18 |       control with that entity. For the purposes of this definition,
 19 |       "control" means (i) the power, direct or indirect, to cause the
 20 |       direction or management of such entity, whether by contract or
 21 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 22 |       outstanding shares, or (iii) beneficial ownership of such entity.
 23 | 
 24 |       "You" (or "Your") shall mean an individual or Legal Entity
 25 |       exercising permissions granted by this License.
 26 | 
 27 |       "Source" form shall mean the preferred form for making modifications,
 28 |       including but not limited to software source code, documentation
 29 |       source, and configuration files.
 30 | 
 31 |       "Object" form shall mean any form resulting from mechanical
 32 |       transformation or translation of a Source form, including but
 33 |       not limited to compiled object code, generated documentation,
 34 |       and conversions to other media types.
 35 | 
 36 |       "Work" shall mean the work of authorship, whether in Source or
 37 |       Object form, made available under the License, as indicated by a
 38 |       copyright notice that is included in or attached to the work
 39 |       (an example is provided in the Appendix below).
 40 | 
 41 |       "Derivative Works" shall mean any work, whether in Source or Object
 42 |       form, that is based on (or derived from) the Work and for which the
 43 |       editorial revisions, annotations, elaborations, or other modifications
 44 |       represent, as a whole, an original work of authorship. For the purposes
 45 |       of this License, Derivative Works shall not include works that remain
 46 |       separable from, or merely link (or bind by name) to the interfaces of,
 47 |       the Work and Derivative Works thereof.
 48 | 
 49 |       "Contribution" shall mean any work of authorship, including
 50 |       the original version of the Work and any modifications or additions
 51 |       to that Work or Derivative Works thereof, that is intentionally
 52 |       submitted to Licensor for inclusion in the Work by the copyright owner
 53 |       or by an individual or Legal Entity authorized to submit on behalf of
 54 |       the copyright owner. For the purposes of this definition, "submitted"
 55 |       means any form of electronic, verbal, or written communication sent
 56 |       to the Licensor or its representatives, including but not limited to
 57 |       communication on electronic mailing lists, source code control systems,
 58 |       and issue tracking systems that are managed by, or on behalf of, the
 59 |       Licensor for the purpose of discussing and improving the Work, but
 60 |       excluding communication that is conspicuously marked or otherwise
 61 |       designated in writing by the copyright owner as "Not a Contribution."
 62 | 
 63 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 64 |       on behalf of whom a Contribution has been received by Licensor and
 65 |       subsequently incorporated within the Work.
 66 | 
 67 |    2. Grant of Copyright License. Subject to the terms and conditions of
 68 |       this License, each Contributor hereby grants to You a perpetual,
 69 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 70 |       copyright license to reproduce, prepare Derivative Works of,
 71 |       publicly display, publicly perform, sublicense, and distribute the
 72 |       Work and such Derivative Works in Source or Object form.
 73 | 
 74 |    3. Grant of Patent License. Subject to the terms and conditions of
 75 |       this License, each Contributor hereby grants to You a perpetual,
 76 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 77 |       (except as stated in this section) patent license to make, have made,
 78 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 79 |       where such license applies only to those patent claims licensable
 80 |       by such Contributor that are necessarily infringed by their
 81 |       Contribution(s) alone or by combination of their Contribution(s)
 82 |       with the Work to which such Contribution(s) was submitted. If You
 83 |       institute patent litigation against any entity (including a
 84 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 85 |       or a Contribution incorporated within the Work constitutes direct
 86 |       or contributory patent infringement, then any patent licenses
 87 |       granted to You under this License for that Work shall terminate
 88 |       as of the date such litigation is filed.
 89 | 
 90 |    4. Redistribution. You may reproduce and distribute copies of the
 91 |       Work or Derivative Works thereof in any medium, with or without
 92 |       modifications, and in Source or Object form, provided that You
 93 |       meet the following conditions:
 94 | 
 95 |       (a) You must give any other recipients of the Work or
 96 |           Derivative Works a copy of this License; and
 97 | 
 98 |       (b) You must cause any modified files to carry prominent notices
 99 |           stating that You changed the files; and
100 | 
101 |       (c) You must retain, in the Source form of any Derivative Works
102 |           that You distribute, all copyright, patent, trademark, and
103 |           attribution notices from the Source form of the Work,
104 |           excluding those notices that do not pertain to any part of
105 |           the Derivative Works; and
106 | 
107 |       (d) If the Work includes a "NOTICE" text file as part of its
108 |           distribution, then any Derivative Works that You distribute must
109 |           include a readable copy of the attribution notices contained
110 |           within such NOTICE file, excluding those notices that do not
111 |           pertain to any part of the Derivative Works, in at least one
112 |           of the following places: within a NOTICE text file distributed
113 |           as part of the Derivative Works; within the Source form or
114 |           documentation, if provided along with the Derivative Works; or,
115 |           within a display generated by the Derivative Works, if and
116 |           wherever such third-party notices normally appear. The contents
117 |           of the NOTICE file are for informational purposes only and
118 |           do not modify the License. You may add Your own attribution
119 |           notices within Derivative Works that You distribute, alongside
120 |           or as an addendum to the NOTICE text from the Work, provided
121 |           that such additional attribution notices cannot be construed
122 |           as modifying the License.
123 | 
124 |       You may add Your own copyright statement to Your modifications and
125 |       may provide additional or different license terms and conditions
126 |       for use, reproduction, or distribution of Your modifications, or
127 |       for any such Derivative Works as a whole, provided Your use,
128 |       reproduction, and distribution of the Work otherwise complies with
129 |       the conditions stated in this License.
130 | 
131 |    5. Submission of Contributions. Unless You explicitly state otherwise,
132 |       any Contribution intentionally submitted for inclusion in the Work
133 |       by You to the Licensor shall be under the terms and conditions of
134 |       this License, without any additional terms or conditions.
135 |       Notwithstanding the above, nothing herein shall supersede or modify
136 |       the terms of any separate license agreement you may have executed
137 |       with Licensor regarding such Contributions.
138 | 
139 |    6. Trademarks. This License does not grant permission to use the trade
140 |       names, trademarks, service marks, or product names of the Licensor,
141 |       except as required for reasonable and customary use in describing the
142 |       origin of the Work and reproducing the content of the NOTICE file.
143 | 
144 |    7. Disclaimer of Warranty. Unless required by applicable law or
145 |       agreed to in writing, Licensor provides the Work (and each
146 |       Contributor provides its Contributions) on an "AS IS" BASIS,
147 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148 |       implied, including, without limitation, any warranties or conditions
149 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150 |       PARTICULAR PURPOSE. You are solely responsible for determining the
151 |       appropriateness of using or redistributing the Work and assume any
152 |       risks associated with Your exercise of permissions under this License.
153 | 
154 |    8. Limitation of Liability. In no event and under no legal theory,
155 |       whether in tort (including negligence), contract, or otherwise,
156 |       unless required by applicable law (such as deliberate and grossly
157 |       negligent acts) or agreed to in writing, shall any Contributor be
158 |       liable to You for damages, including any direct, indirect, special,
159 |       incidental, or consequential damages of any character arising as a
160 |       result of this License or out of the use or inability to use the
161 |       Work (including but not limited to damages for loss of goodwill,
162 |       work stoppage, computer failure or malfunction, or any and all
163 |       other commercial damages or losses), even if such Contributor
164 |       has been advised of the possibility of such damages.
165 | 
166 |    9. Accepting Warranty or Additional Liability. While redistributing
167 |       the Work or Derivative Works thereof, You may choose to offer,
168 |       and charge a fee for, acceptance of support, warranty, indemnity,
169 |       or other liability obligations and/or rights consistent with this
170 |       License. However, in accepting such obligations, You may act only
171 |       on Your own behalf and on Your sole responsibility, not on behalf
172 |       of any other Contributor, and only if You agree to indemnify,
173 |       defend, and hold each Contributor harmless for any liability
174 |       incurred by, or claims asserted against, such Contributor by reason
175 |       of your accepting any such warranty or additional liability.
176 | 
177 |    END OF TERMS AND CONDITIONS
178 | 
179 |    APPENDIX: How to apply the Apache License to your work.
180 | 
181 |       To apply the Apache License to your work, attach the following
182 |       boilerplate notice, with the fields enclosed by brackets "[]"
183 |       replaced with your own identifying information. (Don't include
184 |       the brackets!)  The text should be enclosed in the appropriate
185 |       comment syntax for the file format. We also recommend that a
186 |       file or class name and description of purpose be included on the
187 |       same "printed page" as the copyright notice for easier
188 |       identification within third-party archives.
189 | 
190 |    Copyright [yyyy] [name of copyright owner]
191 | 
192 |    Licensed under the Apache License, Version 2.0 (the "License");
193 |    you may not use this file except in compliance with the License.
194 |    You may obtain a copy of the License at
195 | 
196 |        http://www.apache.org/licenses/LICENSE-2.0
197 | 
198 |    Unless required by applicable law or agreed to in writing, software
199 |    distributed under the License is distributed on an "AS IS" BASIS,
200 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201 |    See the License for the specific language governing permissions and
202 |    limitations under the License.
203 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include .pylintrc
2 | include coverage.sh
3 | include LICENSE
4 | include README.rst
5 | graft doc
6 | prune doc/_build/*
7 | graft test
8 | 


--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
 1 | ..
 2 |   Copyright 2013 craigslist
 3 |  
 4 |   Licensed under the Apache License, Version 2.0 (the "License");
 5 |   you may not use this file except in compliance with the License.
 6 |   You may obtain a copy of the License at
 7 |  
 8 |       http://www.apache.org/licenses/LICENSE-2.0
 9 |  
10 |   Unless required by applicable law or agreed to in writing, software
11 |   distributed under the License is distributed on an "AS IS" BASIS,
12 |   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 |   See the License for the specific language governing permissions and
14 |   limitations under the License.
15 | 
16 | This is the craigslist image python package. It contains an image
17 | processing library, tool, and server. For full documentation, run pydoc
18 | on the command line with package and module names::
19 | 
20 |     pydoc climage
21 | 
22 | An HTML version of the documentation can also be built. Once complete,
23 | point a browser to: doc/_build/html/index.html::
24 | 
25 |     python setup.py build_sphinx
26 | 
27 | To install the package (use --prefix option for a specific location)::
28 | 
29 |     python setup.py install
30 | 
31 | To run the test suite::
32 | 
33 |     python setup.py nosetests
34 | 
35 | If python-coverage is installed, text and HTML code coverage reports can
36 | be generated for the test suite and command line programs by running::
37 | 
38 |     ./coverage.sh
39 | 
40 | There are a number of code style checks already in place using the tools
41 | below. While they are good settings for now, don't hesitate to bring up
42 | any violations encountered for discussion if it should be allowed. To
43 | run the checks::
44 | 
45 |     pylint -iy --rcfile .pylintrc climage test setup.py
46 |     pep8 --ignore=E128 -r .
47 |     pyflakes . | grep -v "undefined name '_'"
48 | 
49 | To build a source tarball for distribution (see 'dist' directory after)::
50 | 
51 |     python setup.py sdist
52 | 


--------------------------------------------------------------------------------
/bin/climageprocessor:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | # Copyright 2013 craigslist
 3 | #
 4 | # Licensed under the Apache License, Version 2.0 (the "License");
 5 | # you may not use this file except in compliance with the License.
 6 | # You may obtain a copy of the License at
 7 | #
 8 | #     http://www.apache.org/licenses/LICENSE-2.0
 9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | # If ../climage/__init__.py exists, add ../ to the Python search path so
17 | # that it will override whatever may be installed in the default Python
18 | # search path.
19 | package_dir=$(cd `dirname "$0"`; cd ..; pwd)
20 | if [ -f "$package_dir/climage/__init__.py" ]
21 | then
22 |     PYTHONPATH="$package_dir:$PYTHONPATH"
23 |     export PYTHONPATH
24 | fi
25 | 
26 | exec /usr/bin/env python -u -m climage.processor "$@"
27 | 


--------------------------------------------------------------------------------
/bin/climageserver:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | # Copyright 2013 craigslist
 3 | #
 4 | # Licensed under the Apache License, Version 2.0 (the "License");
 5 | # you may not use this file except in compliance with the License.
 6 | # You may obtain a copy of the License at
 7 | #
 8 | #     http://www.apache.org/licenses/LICENSE-2.0
 9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | # If ../climage/__init__.py exists, add ../ to the Python search path so
17 | # that it will override whatever may be installed in the default Python
18 | # search path.
19 | package_dir=$(cd `dirname "$0"`; cd ..; pwd)
20 | if [ -f "$package_dir/climage/__init__.py" ]
21 | then
22 |     PYTHONPATH="$package_dir:$PYTHONPATH"
23 |     export PYTHONPATH
24 | fi
25 | 
26 | exec /usr/bin/env python -u -m climage.server "$@"
27 | 


--------------------------------------------------------------------------------
/climage/__init__.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2013 craigslist
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | '''craigslist image package.
16 | 
17 | This provides tools and a server for processing images. Specifically, it
18 | resizes, crops, rotates according to EXIF data tags for multiple sizes,
19 | and then stores it into the blob service. The server provides a simple
20 | HTTP interface for clients to submit the original image and options to.'''
21 | 
22 | # Install the _(...) function as a built-in so all other modules don't need to.
23 | import gettext
24 | gettext.install('climage')
25 | 
26 | __version__ = '0'
27 | 


--------------------------------------------------------------------------------
/climage/exif.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2013 craigslist
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | 
 15 | '''craigslist image exif module.
 16 | 
 17 | Dictionaries for EXIF tag code to name translation.'''
 18 | 
 19 | TAGS = {
 20 |     0x000b: "ProcessingSoftware",
 21 |     0x00fe: "NewSubfileType",
 22 |     0x00ff: "SubfileType",
 23 |     0x0100: "ImageWidth",
 24 |     0x0101: "ImageLength",
 25 |     0x0102: "BitsPerSample",
 26 |     0x0103: "Compression",
 27 |     0x0106: "PhotometricInterpretation",
 28 |     0x0107: "Threshholding",
 29 |     0x0108: "CellWidth",
 30 |     0x0109: "CellLength",
 31 |     0x010a: "FillOrder",
 32 |     0x010d: "DocumentName",
 33 |     0x010e: "ImageDescription",
 34 |     0x010f: "Make",
 35 |     0x0110: "Model",
 36 |     0x0111: "StripOffsets",
 37 |     0x0112: "Orientation",
 38 |     0x0115: "SamplesPerPixel",
 39 |     0x0116: "RowsPerStrip",
 40 |     0x0117: "StripByteCounts",
 41 |     0x011a: "XResolution",
 42 |     0x011b: "YResolution",
 43 |     0x011c: "PlanarConfiguration",
 44 |     0x0122: "GrayResponseUnit",
 45 |     0x0123: "GrayResponseCurve",
 46 |     0x0124: "T4Options",
 47 |     0x0125: "T6Options",
 48 |     0x0128: "ResolutionUnit",
 49 |     0x012d: "TransferFunction",
 50 |     0x0131: "Software",
 51 |     0x0132: "DateTime",
 52 |     0x013b: "Artist",
 53 |     0x013c: "HostComputer",
 54 |     0x013d: "Predictor",
 55 |     0x013e: "WhitePoint",
 56 |     0x013f: "PrimaryChromaticities",
 57 |     0x0140: "ColorMap",
 58 |     0x0141: "HalftoneHints",
 59 |     0x0142: "TileWidth",
 60 |     0x0143: "TileLength",
 61 |     0x0144: "TileOffsets",
 62 |     0x0145: "TileByteCounts",
 63 |     0x014a: "SubIFDs",
 64 |     0x014c: "InkSet",
 65 |     0x014d: "InkNames",
 66 |     0x014e: "NumberOfInks",
 67 |     0x0150: "DotRange",
 68 |     0x0151: "TargetPrinter",
 69 |     0x0152: "ExtraSamples",
 70 |     0x0153: "SampleFormat",
 71 |     0x0154: "SMinSampleValue",
 72 |     0x0155: "SMaxSampleValue",
 73 |     0x0156: "TransferRange",
 74 |     0x0157: "ClipPath",
 75 |     0x0158: "XClipPathUnits",
 76 |     0x0159: "YClipPathUnits",
 77 |     0x015a: "Indexed",
 78 |     0x015b: "JPEGTables",
 79 |     0x015f: "OPIProxy",
 80 |     0x0200: "JPEGProc",
 81 |     0x0201: "JPEGInterchangeFormat",
 82 |     0x0202: "JPEGInterchangeFormatLength",
 83 |     0x0203: "JPEGRestartInterval",
 84 |     0x0205: "JPEGLosslessPredictors",
 85 |     0x0206: "JPEGPointTransforms",
 86 |     0x0207: "JPEGQTables",
 87 |     0x0208: "JPEGDCTables",
 88 |     0x0209: "JPEGACTables",
 89 |     0x0211: "YCbCrCoefficients",
 90 |     0x0212: "YCbCrSubSampling",
 91 |     0x0213: "YCbCrPositioning",
 92 |     0x0214: "ReferenceBlackWhite",
 93 |     0x02bc: "XMLPacket",
 94 |     0x4746: "Rating",
 95 |     0x4749: "RatingPercent",
 96 |     0x800d: "ImageID",
 97 |     0x828d: "CFARepeatPatternDim",
 98 |     0x828e: "CFAPattern",
 99 |     0x828f: "BatteryLevel",
100 |     0x8298: "Copyright",
101 |     0x829a: "ExposureTime",
102 |     0x829d: "FNumber",
103 |     0x83bb: "IPTCNAA",
104 |     0x8649: "ImageResources",
105 |     0x8769: "ExifTag",
106 |     0x8773: "InterColorProfile",
107 |     0x8822: "ExposureProgram",
108 |     0x8824: "SpectralSensitivity",
109 |     0x8825: "GPSTag",
110 |     0x8827: "ISOSpeedRatings",
111 |     0x8828: "OECF",
112 |     0x8829: "Interlace",
113 |     0x882a: "TimeZoneOffset",
114 |     0x882b: "SelfTimerMode",
115 |     0x8830: "SensitivityType",
116 |     0x8831: "StandardOutputSensitivity",
117 |     0x8832: "RecommendedExposureIndex",
118 |     0x8833: "ISOSpeed",
119 |     0x8834: "ISOSpeedLatitudeyyy",
120 |     0x8835: "ISOSpeedLatitudezzz",
121 |     0x9000: "ExifVersion",
122 |     0x9003: "DateTimeOriginal",
123 |     0x9004: "DateTimeDigitized",
124 |     0x9101: "ComponentsConfiguration",
125 |     0x9102: "CompressedBitsPerPixel",
126 |     0x9201: "ShutterSpeedValue",
127 |     0x9202: "ApertureValue",
128 |     0x9203: "BrightnessValue",
129 |     0x9204: "ExposureBiasValue",
130 |     0x9205: "MaxApertureValue",
131 |     0x9206: "SubjectDistance",
132 |     0x9207: "MeteringMode",
133 |     0x9208: "LightSource",
134 |     0x9209: "Flash",
135 |     0x920a: "FocalLength",
136 |     0x920b: "FlashEnergy",
137 |     0x920c: "SpatialFrequencyResponse",
138 |     0x920d: "Noise",
139 |     0x920e: "FocalPlaneXResolution",
140 |     0x920f: "FocalPlaneYResolution",
141 |     0x9210: "FocalPlaneResolutionUnit",
142 |     0x9211: "ImageNumber",
143 |     0x9212: "SecurityClassification",
144 |     0x9213: "ImageHistory",
145 |     0x9214: "SubjectLocation",
146 |     0x9215: "ExposureIndex",
147 |     0x9216: "TIFFEPStandardID",
148 |     0x9217: "SensingMethod",
149 |     0x927c: "MakerNote",
150 |     0x9286: "UserComment",
151 |     0x9290: "SubSecTime",
152 |     0x9291: "SubSecTimeOriginal",
153 |     0x9292: "SubSecTimeDigitized",
154 |     0x9c9b: "XPTitle",
155 |     0x9c9c: "XPComment",
156 |     0x9c9d: "XPAuthor",
157 |     0x9c9e: "XPKeywords",
158 |     0x9c9f: "XPSubject",
159 |     0xa000: "FlashpixVersion",
160 |     0xa001: "ColorSpace",
161 |     0xa002: "PixelXDimension",
162 |     0xa003: "PixelYDimension",
163 |     0xa004: "RelatedSoundFile",
164 |     0xa005: "InteroperabilityTag",
165 |     0xa20b: "FlashEnergy",
166 |     0xa20c: "SpatialFrequencyResponse",
167 |     0xa20e: "FocalPlaneXResolution",
168 |     0xa20f: "FocalPlaneYResolution",
169 |     0xa210: "FocalPlaneResolutionUnit",
170 |     0xa214: "SubjectLocation",
171 |     0xa215: "ExposureIndex",
172 |     0xa217: "SensingMethod",
173 |     0xa300: "FileSource",
174 |     0xa301: "SceneType",
175 |     0xa302: "CFAPattern",
176 |     0xa401: "CustomRendered",
177 |     0xa402: "ExposureMode",
178 |     0xa403: "WhiteBalance",
179 |     0xa404: "DigitalZoomRatio",
180 |     0xa405: "FocalLengthIn35mmFilm",
181 |     0xa406: "SceneCaptureType",
182 |     0xa407: "GainControl",
183 |     0xa408: "Contrast",
184 |     0xa409: "Saturation",
185 |     0xa40a: "Sharpness",
186 |     0xa40b: "DeviceSettingDescription",
187 |     0xa40c: "SubjectDistanceRange",
188 |     0xa420: "ImageUniqueID",
189 |     0xa430: "CameraOwnerName",
190 |     0xa431: "BodySerialNumber",
191 |     0xa432: "LensSpecification",
192 |     0xa433: "LensMake",
193 |     0xa434: "LensModel",
194 |     0xa435: "LensSerialNumber",
195 |     0xc4a5: "PrintImageMatching",
196 |     0xc612: "DNGVersion",
197 |     0xc613: "DNGBackwardVersion",
198 |     0xc614: "UniqueCameraModel",
199 |     0xc615: "LocalizedCameraModel",
200 |     0xc616: "CFAPlaneColor",
201 |     0xc617: "CFALayout",
202 |     0xc618: "LinearizationTable",
203 |     0xc619: "BlackLevelRepeatDim",
204 |     0xc61a: "BlackLevel",
205 |     0xc61b: "BlackLevelDeltaH",
206 |     0xc61c: "BlackLevelDeltaV",
207 |     0xc61d: "WhiteLevel",
208 |     0xc61e: "DefaultScale",
209 |     0xc61f: "DefaultCropOrigin",
210 |     0xc620: "DefaultCropSize",
211 |     0xc621: "ColorMatrix1",
212 |     0xc622: "ColorMatrix2",
213 |     0xc623: "CameraCalibration1",
214 |     0xc624: "CameraCalibration2",
215 |     0xc625: "ReductionMatrix1",
216 |     0xc626: "ReductionMatrix2",
217 |     0xc627: "AnalogBalance",
218 |     0xc628: "AsShotNeutral",
219 |     0xc629: "AsShotWhiteXY",
220 |     0xc62a: "BaselineExposure",
221 |     0xc62b: "BaselineNoise",
222 |     0xc62c: "BaselineSharpness",
223 |     0xc62d: "BayerGreenSplit",
224 |     0xc62e: "LinearResponseLimit",
225 |     0xc62f: "CameraSerialNumber",
226 |     0xc630: "LensInfo",
227 |     0xc631: "ChromaBlurRadius",
228 |     0xc632: "AntiAliasStrength",
229 |     0xc633: "ShadowScale",
230 |     0xc634: "DNGPrivateData",
231 |     0xc635: "MakerNoteSafety",
232 |     0xc65a: "CalibrationIlluminant1",
233 |     0xc65b: "CalibrationIlluminant2",
234 |     0xc65c: "BestQualityScale",
235 |     0xc65d: "RawDataUniqueID",
236 |     0xc68b: "OriginalRawFileName",
237 |     0xc68c: "OriginalRawFileData",
238 |     0xc68d: "ActiveArea",
239 |     0xc68e: "MaskedAreas",
240 |     0xc68f: "AsShotICCProfile",
241 |     0xc690: "AsShotPreProfileMatrix",
242 |     0xc691: "CurrentICCProfile",
243 |     0xc692: "CurrentPreProfileMatrix",
244 |     0xc6bf: "ColorimetricReference",
245 |     0xc6f3: "CameraCalibrationSignature",
246 |     0xc6f4: "ProfileCalibrationSignature",
247 |     0xc6f6: "AsShotProfileName",
248 |     0xc6f7: "NoiseReductionApplied",
249 |     0xc6f8: "ProfileName",
250 |     0xc6f9: "ProfileHueSatMapDims",
251 |     0xc6fa: "ProfileHueSatMapData1",
252 |     0xc6fb: "ProfileHueSatMapData2",
253 |     0xc6fc: "ProfileToneCurve",
254 |     0xc6fd: "ProfileEmbedPolicy",
255 |     0xc6fe: "ProfileCopyright",
256 |     0xc714: "ForwardMatrix1",
257 |     0xc715: "ForwardMatrix2",
258 |     0xc716: "PreviewApplicationName",
259 |     0xc717: "PreviewApplicationVersion",
260 |     0xc718: "PreviewSettingsName",
261 |     0xc719: "PreviewSettingsDigest",
262 |     0xc71a: "PreviewColorSpace",
263 |     0xc71b: "PreviewDateTime",
264 |     0xc71c: "RawImageDigest",
265 |     0xc71d: "OriginalRawFileDigest",
266 |     0xc71e: "SubTileBlockSize",
267 |     0xc71f: "RowInterleaveFactor",
268 |     0xc725: "ProfileLookTableDims",
269 |     0xc726: "ProfileLookTableData",
270 |     0xc740: "OpcodeList1",
271 |     0xc741: "OpcodeList2",
272 |     0xc74e: "OpcodeList3",
273 |     0xc761: "NoiseProfile"}
274 | 
275 | IOP_TAGS = {
276 |     0x0001: "InteroperabilityIndex",
277 |     0x0002: "InteroperabilityVersion",
278 |     0x1000: "RelatedImageFileFormat",
279 |     0x1001: "RelatedImageWidth",
280 |     0x1002: "RelatedImageLength"}
281 | 
282 | GPSINFO_TAGS = {
283 |     0x0000: "GPSVersionID",
284 |     0x0001: "GPSLatitudeRef",
285 |     0x0002: "GPSLatitude",
286 |     0x0003: "GPSLongitudeRef",
287 |     0x0004: "GPSLongitude",
288 |     0x0005: "GPSAltitudeRef",
289 |     0x0006: "GPSAltitude",
290 |     0x0007: "GPSTimeStamp",
291 |     0x0008: "GPSSatellites",
292 |     0x0009: "GPSStatus",
293 |     0x000a: "GPSMeasureMode",
294 |     0x000b: "GPSDOP",
295 |     0x000c: "GPSSpeedRef",
296 |     0x000d: "GPSSpeed",
297 |     0x000e: "GPSTrackRef",
298 |     0x000f: "GPSTrack",
299 |     0x0010: "GPSImgDirectionRef",
300 |     0x0011: "GPSImgDirection",
301 |     0x0012: "GPSMapDatum",
302 |     0x0013: "GPSDestLatitudeRef",
303 |     0x0014: "GPSDestLatitude",
304 |     0x0015: "GPSDestLongitudeRef",
305 |     0x0016: "GPSDestLongitude",
306 |     0x0017: "GPSDestBearingRef",
307 |     0x0018: "GPSDestBearing",
308 |     0x0019: "GPSDestDistanceRef",
309 |     0x001a: "GPSDestDistance",
310 |     0x001b: "GPSProcessingMethod",
311 |     0x001c: "GPSAreaInformation",
312 |     0x001d: "GPSDateStamp",
313 |     0x001e: "GPSDifferential"}
314 | 


--------------------------------------------------------------------------------
/climage/processor.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2013 craigslist
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | 
 15 | '''craigslist image processor module.
 16 | 
 17 | This provides a processor class that can be run on the command line or
 18 | called through the server module.'''
 19 | 
 20 | import hashlib
 21 | import json
 22 | import pgmagick
 23 | import PIL.Image
 24 | import PIL.ImageFile
 25 | import re
 26 | import StringIO
 27 | import sys
 28 | import time
 29 | 
 30 | import clblob.client
 31 | import clcommon.config
 32 | import clcommon.log
 33 | import clcommon.profile
 34 | import clcommon.worker
 35 | import climage.exif
 36 | 
 37 | # Increase max blocks in ImageFile lib to allow for saving larger images.
 38 | PIL.ImageFile.MAXBLOCK = 1048576
 39 | 
 40 | DEFAULT_CONFIG = clcommon.config.update(clblob.client.DEFAULT_CONFIG, {
 41 |     'climage': {
 42 |         'processor': {
 43 |             'formats': ['TIFF', 'BMP', 'JPEG', 'GIF', 'PNG'],
 44 |             'log_level': 'NOTSET',
 45 |             'max_height': 7000,
 46 |             'max_width': 7000,
 47 |             'pool_size': 8,
 48 |             'quality': 70,
 49 |             'save': True,
 50 |             'save_blob': True,
 51 |             'sizes': ['50x50c', '300x300', '600x450'],
 52 |             'ttl': 7776000}}})  # 90 days
 53 | 
 54 | DEFAULT_CONFIG_FILES = clblob.client.DEFAULT_CONFIG_FILES + [
 55 |     '/etc/climageprocessor.conf',
 56 |     '~/.climageprocessor.conf']
 57 | DEFAULT_CONFIG_DIRS = clblob.client.DEFAULT_CONFIG_DIRS + [
 58 |     '/etc/climageprocessor.d',
 59 |     '~/.climageprocessor.d']
 60 | 
 61 | SIZE_REGEX = re.compile('^([0-9]+)x([0-9]+)(.*)')
 62 | 
 63 | ORIENTATION_OPERATIONS = {
 64 |     1: [],
 65 |     2: [PIL.Image.FLIP_LEFT_RIGHT],
 66 |     3: [PIL.Image.ROTATE_180],
 67 |     4: [PIL.Image.FLIP_TOP_BOTTOM],
 68 |     5: [PIL.Image.FLIP_TOP_BOTTOM, PIL.Image.ROTATE_270],
 69 |     6: [PIL.Image.ROTATE_270],
 70 |     7: [PIL.Image.FLIP_LEFT_RIGHT, PIL.Image.ROTATE_270],
 71 |     8: [PIL.Image.ROTATE_90]}
 72 | 
 73 | 
 74 | class Processor(object):
 75 |     '''Image processing class. This handles a processing job for a single
 76 |     image. An optional worker pool and blob client can be passed in for
 77 |     use between different processor objects.'''
 78 | 
 79 |     def __init__(self, config, image, pool=None, blob_client=None):
 80 |         self.config = config['climage']['processor']
 81 |         self._pool = pool or clcommon.worker.Pool(self.config['pool_size'])
 82 |         self._stop_pool = pool is None
 83 |         if self.config['save_blob'] and blob_client is None:
 84 |             blob_client = clblob.client.Client(config)
 85 |         self._blob_client = blob_client
 86 |         self.log = clcommon.log.get_log('climage_processor',
 87 |             self.config['log_level'])
 88 |         self.profile = clcommon.profile.Profile()
 89 |         if not isinstance(image, str):
 90 |             image = image.read()
 91 |             self.profile.mark_time('read')
 92 |         self.raw = image
 93 |         self.profile.mark('original_size', len(self.raw))
 94 |         self._pgmagick_ran = False
 95 |         self._processed = {}
 96 |         self.info = {}
 97 |         self._orientation = 1
 98 |         self._sizes = []
 99 |         for size in self.config['sizes']:
100 |             match = SIZE_REGEX.match(size)
101 |             if match is None:
102 |                 raise ProcessingError(_('Invalid size parameter: %s') % size)
103 |             parsed_size = dict(name=size)
104 |             parsed_size['width'] = int(match.group(1))
105 |             parsed_size['height'] = int(match.group(2))
106 |             parsed_size['flags'] = match.group(3)
107 |             self._sizes.append(parsed_size)
108 | 
109 |     def __del__(self):
110 |         if hasattr(self, '_pool') and self._stop_pool:
111 |             self._pool.stop()
112 |         if hasattr(self, 'profile') and len(self.profile.marks) > 0:
113 |             self.log.info('profile %s', self.profile)
114 | 
115 |     def process(self):
116 |         '''Process the image as specified in the config. Image info
117 |         (such as the blob names after being saved) can be found in the
118 |         info attribute when this returns. This returns a dictionary of
119 |         resized images, indexed by the size name from the config.'''
120 |         self.profile.reset_time()
121 |         start = time.time()
122 |         image = self._pool.start(self._load).wait()
123 |         batch = self._pool.batch()
124 |         for size in self._sizes:
125 |             batch.start(self._process, size, image)
126 |             image = None
127 |         batch.wait_all()
128 |         self.profile.reset_time()
129 |         if self.config['save']:
130 |             if self.config['save_blob']:
131 |                 self._save_blob()
132 |         self.profile.mark('real_time', time.time() - start)
133 |         return self._processed
134 | 
135 |     def _load(self):
136 |         '''Load image and parse info.'''
137 |         try:
138 |             image = PIL.Image.open(StringIO.StringIO(self.raw))
139 |         except Exception:
140 |             self.profile.mark_time('open')
141 |             try:
142 |                 self._pgmagick()
143 |                 image = PIL.Image.open(StringIO.StringIO(self.raw))
144 |             except Exception, exception:
145 |                 raise BadImage(_('Cannot open image: %s') % exception)
146 |         self.profile.mark_time('open')
147 | 
148 |         self._get_info(image)
149 |         self._check_info(self.info)
150 | 
151 |         self._orientation = int(self.info.get('exif_orientation', 1))
152 |         if self._orientation not in ORIENTATION_OPERATIONS:
153 |             self._orientation = 1
154 | 
155 |         if len(self._sizes) == 0:
156 |             return image
157 | 
158 |         # Fix width and height to keep aspect ration for non-cropped images.
159 |         for size in self._sizes:
160 |             if 'c' in size['flags']:
161 |                 continue
162 |             width, height = image.size
163 |             if self._orientation > 4:
164 |                 # Width and height will be reversed for these orientations.
165 |                 width, height = height, width
166 |             if width > size['width']:
167 |                 height = max(height * size['width'] / width, 1)
168 |                 width = size['width']
169 |             if height > size['height']:
170 |                 width = max(width * size['height'] / height, 1)
171 |                 height = size['height']
172 |             size['width'], size['height'] = width, height
173 | 
174 |         try:
175 |             self._load_image(image, self._sizes[0])
176 |         except Exception:
177 |             self.profile.mark_time('load')
178 |             try:
179 |                 self._pgmagick()
180 |                 image = PIL.Image.open(StringIO.StringIO(self.raw))
181 |                 self.profile.mark_time('open')
182 |                 self._load_image(image, self._sizes[0])
183 |             except Exception, exception:
184 |                 raise BadImage(_('Cannot load image: %s') % exception)
185 |         self.profile.mark_time('load')
186 | 
187 |         return image
188 | 
189 |     def _load_image(self, image, size):
190 |         '''Load the image using the smallest sample we can.'''
191 |         width, height = size['width'], size['height']
192 |         if self._orientation > 4:
193 |             # Width and height will be reversed for these orientations.
194 |             width, height = height, width
195 |         image.draft(None, (width, height))
196 |         image.load()
197 | 
198 |     def _get_info(self, image):
199 |         '''Parse out all info and exif data embedded in image.'''
200 |         for key, value in image.info.iteritems():
201 |             if key == 'exif' and hasattr(image, '_getexif'):
202 |                 self._get_exif(image)
203 |             else:
204 |                 self._set_info(key, value)
205 |         if 'filename' in self.config:
206 |             self.info['filename'] = self.config['filename']
207 |         self.info['width'] = image.size[0]
208 |         self.info['height'] = image.size[1]
209 |         self.info['format'] = image.format
210 |         self.info['mode'] = image.mode
211 |         self.profile.mark_time('info')
212 |         value = hashlib.sha256(self.raw).hexdigest()  # pylint: disable=E1101
213 |         self.info['checksum'] = value
214 |         self.profile.mark_time('checksum')
215 | 
216 |     def _get_exif(self, image):
217 |         '''Add exif data to the info dictionary.'''
218 |         try:
219 |             exif = image._getexif()  # pylint: disable=W0212
220 |         except Exception:
221 |             return
222 |         for key, value in exif.items():
223 |             key = climage.exif.TAGS.get(key, key)
224 |             if key == 'GPSTag':
225 |                 for gps_key, gps_value in value.items():
226 |                     gps_key = climage.exif.GPSINFO_TAGS.get(gps_key, gps_key)
227 |                     self._set_info('exif_%s' % gps_key, gps_value)
228 |             else:
229 |                 self._set_info('exif_%s' % key, value)
230 | 
231 |     def _set_info(self, key, value):
232 |         '''Set an info key value pair if it is UTF-8 safe.'''
233 |         try:
234 |             value = str(value)
235 |             value = value.replace('\x00', '')
236 |             value.encode('utf-8')
237 |             if len(value) > 1024:
238 |                 self.log.debug(_('Value too large for info key: %s(%d'), key,
239 |                     len(value))
240 |                 return
241 |             self.info[str(key).lower()] = value
242 |         except UnicodeError:
243 |             self.log.debug(_('Value not UTF-8 safe for info key: %s'), key)
244 | 
245 |     def _check_info(self, info):
246 |         '''Make sure image is allowed with given info.'''
247 |         if info['format'] == '':
248 |             raise BadImage(_('Unknown image format'))
249 |         if info['format'] not in self.config['formats']:
250 |             raise BadImage(_('Invalid image format: %s') % info['format'])
251 |         if info['width'] > self.config['max_width'] or \
252 |                 info['height'] > self.config['max_height']:
253 |             raise BadImage(_('Image too large: %dx%d') %
254 |                 (info['width'], info['height']))
255 | 
256 |     def _process(self, size, image=None):
257 |         '''Process a given image size.'''
258 |         profile = clcommon.profile.Profile()
259 | 
260 |         # Used image.copy originally, but that was actually much slower than
261 |         # reopening unless the image has already been modified in some way.
262 |         if image is None:
263 |             image = PIL.Image.open(StringIO.StringIO(self.raw))
264 |             profile.mark_time('open')
265 |             try:
266 |                 self._load_image(image, size)
267 |             except Exception, exception:
268 |                 profile.mark_time('load')
269 |                 raise BadImage(_('Cannot load image (proc): %s') % exception)
270 |             profile.mark_time('load')
271 | 
272 |         width, height = size['width'], size['height']
273 |         if 'c' in size['flags']:
274 |             image = self._crop(image, width, height)
275 |             profile.mark_time('%s:crop' % size['name'])
276 | 
277 |         if self._orientation > 4:
278 |             # Width and height will be reversed for these orientations.
279 |             width, height = height, width
280 |         image = image.resize((width, height), PIL.Image.ANTIALIAS)
281 |         profile.mark_time('%s:resize' % size['name'])
282 | 
283 |         if self._orientation > 1:
284 |             for operation in ORIENTATION_OPERATIONS[self._orientation]:
285 |                 image = image.transpose(operation)
286 |             profile.mark_time('%s:transpose' % size['name'])
287 | 
288 |         if image.mode in ['P', 'LA']:
289 |             image = image.convert(mode='RGB')
290 |             profile.mark_time('%s:convert' % size['name'])
291 | 
292 |         output = StringIO.StringIO()
293 |         image.save(output, 'JPEG', quality=self.config['quality'],
294 |             optimize=True)
295 |         raw = output.getvalue()
296 |         self._processed[size['name']] = raw
297 |         profile.mark_time('%s:save' % size['name'])
298 |         profile.mark('%s:size' % size['name'], len(raw))
299 |         self.profile.update(profile)
300 | 
301 |     def _crop(self, image, end_width, end_height):
302 |         '''Crop the image if needed.'''
303 |         width, height = image.size
304 |         if self._orientation > 4:
305 |             # Width and height will be reversed for these orientations.
306 |             width, height = height, width
307 |         factor = min(float(width) / end_width, float(height) / end_height)
308 |         crop_width = int(end_width * factor)
309 |         crop_height = int(end_height * factor)
310 |         left = (width - crop_width) / 2
311 |         upper = (height - crop_height) / 2
312 |         right = left + crop_width
313 |         lower = upper + crop_height
314 |         if self._orientation > 4:
315 |             left, upper = upper, left
316 |             right, lower = lower, right
317 |         return image.crop((left, upper, right, lower))
318 | 
319 |     def _pgmagick(self):
320 |         '''When an error is encountered while opening an image, run
321 |         it through pgmagick since it is a lot more forgiving of errors
322 |         (truncation, bad headers, etc). This seems to be rare, but this
323 |         way we can process more things successfully. We want to still
324 |         use PIL for all other operations we perform since they are faster
325 |         than pgmagick.'''
326 |         if self._pgmagick_ran:
327 |             raise BadImage(_('Already converted with pgmagick'))
328 |         self._pgmagick_ran = True
329 |         blob = pgmagick.Blob(self.raw)
330 |         image = pgmagick.Image()
331 |         image.ping(blob)
332 |         self._check_info(dict(format=image.magick(), width=image.columns(),
333 |             height=image.rows()))
334 |         image = pgmagick.Image(blob)
335 |         image.quality(self.config['quality'])
336 |         blob = pgmagick.Blob()
337 |         image.write(blob)
338 |         self.raw = blob.data
339 |         self.profile.mark_time('pgmagick')
340 |         self.profile.mark('pgmagick_size', len(self.raw))
341 | 
342 |     def _save_blob(self):
343 |         '''Save the image to the blob service.'''
344 |         checksum = int(self.info['checksum'][:16], 16)
345 |         checksum = clcommon.anybase.encode(checksum, 62)
346 |         name = self._blob_client.name(checksum)
347 |         pool = clcommon.worker.Pool(4, True)
348 |         batch = pool.batch()
349 |         batch.start(self._blob_client.put, '%s.json' % name,
350 |             json.dumps(self.info), self.config['ttl'], encoded=True)
351 |         self.info['blob_info_name'] = '%s.json' % name
352 |         self.info['blob_names'] = {}
353 |         for size in self._processed:
354 |             batch.start(self._blob_client.put, '%s_%s.jpg' % (name, size),
355 |                 self._processed[size], self.config['ttl'], encoded=True)
356 |             self.info['blob_names'][size] = '%s_%s.jpg' % (name, size)
357 |         batch.wait_all()
358 |         pool.stop()
359 |         self.log.info('save_blob_name: %s', name)
360 |         self.profile.mark_time('save_blob')
361 | 
362 | 
363 | class ProcessingError(Exception):
364 |     '''Exception raised when a processing error is encountered.'''
365 | 
366 |     pass
367 | 
368 | 
369 | class BadImage(Exception):
370 |     '''Exception raised when a bad image is encountered.'''
371 | 
372 |     pass
373 | 
374 | 
375 | def _main():
376 |     '''Run the image tool.'''
377 |     config = clcommon.config.update(DEFAULT_CONFIG,
378 |         clcommon.log.DEFAULT_CONFIG)
379 |     config, filenames = clcommon.config.load(config, DEFAULT_CONFIG_FILES,
380 |         DEFAULT_CONFIG_DIRS)
381 |     clcommon.log.setup(config)
382 |     if len(filenames) == 0:
383 |         filenames = ['-']
384 |     for filename in filenames:
385 |         if filename == '-':
386 |             image = sys.stdin
387 |         else:
388 |             config = clcommon.config.update_option(config,
389 |                 'climage.processor.filename', filename)
390 |             print filename
391 |             image = open(filename)
392 |         processor = Processor(config, image)
393 |         processed = processor.process()
394 |         for key in processed:
395 |             print '%s: %s' % (key, len(processed[key]))
396 |         print
397 |         for key in sorted(processor.info):
398 |             print '%s: %s' % (key, processor.info[key])
399 |         print
400 |         print
401 | 
402 | 
403 | if __name__ == '__main__':
404 |     _main()
405 | 


--------------------------------------------------------------------------------
/climage/server.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2013 craigslist
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | 
 15 | '''craigslist image server module.
 16 | 
 17 | This is a thin HTTP server layer around the image processor class. This
 18 | adds the ability to save any failed images for later inspection and to
 19 | return either the info or any size image that was requested after being
 20 | processed. This maintains a worker pool and blob client that is shared
 21 | between all requests.'''
 22 | 
 23 | import json
 24 | import os
 25 | import time
 26 | 
 27 | import clblob.client
 28 | import clcommon.config
 29 | import clcommon.http
 30 | import clcommon.server
 31 | import clcommon.worker
 32 | import climage.processor
 33 | 
 34 | DEFAULT_CONFIG = clcommon.config.update(climage.processor.DEFAULT_CONFIG,
 35 |     clcommon.http.DEFAULT_CONFIG)
 36 | DEFAULT_CONFIG = clcommon.config.update(DEFAULT_CONFIG, {
 37 |     'climage': {
 38 |         'server': {
 39 |             'response': 'checksum',
 40 |             'save_bad_path': None}}})
 41 | 
 42 | DEFAULT_CONFIG_FILES = climage.processor.DEFAULT_CONFIG_FILES + [
 43 |     '/etc/climageserver.conf',
 44 |     '~/.climageserver.conf']
 45 | DEFAULT_CONFIG_DIRS = climage.processor.DEFAULT_CONFIG_DIRS + [
 46 |     '/etc/climageserver.d',
 47 |     '~/.climageserver.d']
 48 | 
 49 | 
 50 | VALID_RESPONSES = ['none', 'checksum', 'info']
 51 | 
 52 | 
 53 | class Request(clcommon.http.Request):
 54 |     '''Request handler for image processing.'''
 55 | 
 56 |     def run(self):
 57 |         '''Run the request.'''
 58 |         if self.method not in ['POST', 'PUT']:
 59 |             raise clcommon.http.MethodNotAllowed()
 60 |         config = self.parse_params(['filename'], ['quality', 'ttl'],
 61 |             ['save', 'save_blob'], ['sizes'])
 62 |         config = clcommon.config.update(self.server.config,
 63 |             {'climage': {'processor': config}})
 64 |         response = config['climage']['server']['response']
 65 |         response = self.params.get('response', response).lower()
 66 |         sizes = config['climage']['processor']['sizes']
 67 |         if response not in VALID_RESPONSES + sizes:
 68 |             raise clcommon.http.BadRequest(
 69 |                 _('Invalid response parameter: %s') % response)
 70 |         try:
 71 |             processor = climage.processor.Processor(config, self.body_data,
 72 |                 self.server.image_processor_pool, self.server.blob_client)
 73 |             processed = processor.process()
 74 |         except climage.processor.ProcessingError, exception:
 75 |             raise clcommon.http.BadRequest(str(exception))
 76 |         except climage.processor.BadImage, exception:
 77 |             self.log.warning(_('Bad image file: %s%s'), exception,
 78 |                 self._save_bad())
 79 |             raise clcommon.http.UnsupportedMediaType(_('Bad image file'))
 80 |         body = None
 81 |         if response == 'checksum':
 82 |             body = processor.info['checksum']
 83 |             self.headers.append(('Content-type', 'text/plain'))
 84 |         elif response == 'info':
 85 |             body = json.dumps(processor.info)
 86 |             self.headers.append(('Content-type', 'application/json'))
 87 |         elif response in sizes:
 88 |             body = processed[response]
 89 |             self.headers.append(('Content-type', 'image/jpeg'))
 90 |         return self.ok(body)
 91 | 
 92 |     def _save_bad(self):
 93 |         '''Save bad image file to some location if enabled.'''
 94 |         path = self.server.config['climage']['server']['save_bad_path']
 95 |         if path is None:
 96 |             return ''
 97 |         try:
 98 |             if not os.path.isdir(path):
 99 |                 os.makedirs(path)
100 |             filename = self.params.get('filename', '')
101 |             filename = ''.join(char for char in filename
102 |                 if 32 < ord(char) < 127 and char != '/')
103 |             filename = '%f.%s' % (time.time(), filename)
104 |             filename = os.path.join(path, filename[:100])
105 |             bad_file = open(filename, 'w')
106 |             bad_file.write(self.body_data)
107 |             bad_file.close()
108 |             return ' (%s)' % filename
109 |         except Exception, exception:
110 |             self.log.warning(_('Could not save bad file: %s'), exception)
111 |             return ''
112 | 
113 | 
114 | class Server(clcommon.http.Server):
115 |     '''Wrapper for the HTTP server that adds an image processing pool so
116 |     we can use it across all requests.'''
117 | 
118 |     def __init__(self, config, request):
119 |         super(Server, self).__init__(config, request)
120 |         self.blob_client = None
121 |         self.image_processor_pool = None
122 | 
123 |     def start(self):
124 |         if self.config['climage']['processor']['save_blob']:
125 |             self.blob_client = clblob.client.Client(self.config)
126 |         self.image_processor_pool = clcommon.worker.Pool(
127 |             self.config['climage']['processor']['pool_size'])
128 |         super(Server, self).start()
129 | 
130 |     def stop(self, timeout=None):
131 |         super(Server, self).stop(timeout)
132 |         if self.blob_client is not None:
133 |             self.blob_client.stop()
134 |             self.blob_client = None
135 |         self.image_processor_pool.stop()
136 |         self.image_processor_pool = None
137 | 
138 | 
139 | if __name__ == '__main__':
140 |     clcommon.server.Server(DEFAULT_CONFIG, DEFAULT_CONFIG_FILES,
141 |         DEFAULT_CONFIG_DIRS, [lambda config: Server(config, Request)]).start()
142 | 


--------------------------------------------------------------------------------
/coverage.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/sh
 2 | # Copyright 2013 craigslist
 3 | #
 4 | # Licensed under the Apache License, Version 2.0 (the "License");
 5 | # you may not use this file except in compliance with the License.
 6 | # You may obtain a copy of the License at
 7 | #
 8 | #     http://www.apache.org/licenses/LICENSE-2.0
 9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | echo "+++ Preparing to run coverage"
17 | coverage=`which coverage`
18 | if [ -z $coverage ]; then
19 |     coverage=`which python-coverage`
20 |     if [ -z $coverage ]; then
21 |         echo 'Python coverage not found'
22 |         exit 1
23 |     fi
24 | fi
25 | 
26 | cd `dirname "$0"`
27 | export PYTHONPATH=".:$PYTHONPATH"
28 | rm -rf coverage.html .coverage*
29 | echo
30 | 
31 | echo "+++ Running test suite"
32 | $coverage run -p setup.py nosetests
33 | echo
34 | 
35 | echo "+++ Running commands"
36 | image_config='
37 |     --climage.processor.save=false
38 |     --climage.processor.save_blob=false'
39 | $coverage run -p climage/processor.py -n $image_config test/test.jpg
40 | $coverage run -p climage/processor.py -n $image_config < test/test.jpg
41 | echo
42 | 
43 | for signal in 2 9 15; do
44 |     echo "+++ Testing climageserver shutdown with kill -$signal"
45 |     $coverage run -p climage/server.py -n $image_config \
46 |         --clcommon.http.port=12342 \
47 |         --clcommon.log.syslog_ident=test \
48 |         --clcommon.server.daemonize=true \
49 |         --clcommon.server.pid_file=test_pid
50 |     sleep 0.2
51 |     kill -$signal `cat test_pid`
52 |     sleep 1.2
53 |     rm test_pid
54 | done
55 | echo
56 | 
57 | echo "+++ Generating coverage report"
58 | $coverage combine
59 | $coverage html -d coverage.html --include='climage/*'
60 | $coverage report --include='climage/*'
61 | 


--------------------------------------------------------------------------------
/doc/_build/.gitignore:
--------------------------------------------------------------------------------
1 | *
2 | !.gitignore
3 | 


--------------------------------------------------------------------------------
/doc/climage.exif.rst:
--------------------------------------------------------------------------------
1 | climage.exif
2 | ************
3 | 
4 | .. automodule:: climage.exif
5 |     :members:
6 |     :undoc-members:
7 |     :show-inheritance:
8 | 


--------------------------------------------------------------------------------
/doc/climage.processor.rst:
--------------------------------------------------------------------------------
1 | climage.processor
2 | *****************
3 | 
4 | .. automodule:: climage.processor
5 |     :members:
6 |     :undoc-members:
7 |     :show-inheritance:
8 | 


--------------------------------------------------------------------------------
/doc/climage.server.rst:
--------------------------------------------------------------------------------
1 | climage.server
2 | **************
3 | 
4 | .. automodule:: climage.server
5 |     :members:
6 |     :undoc-members:
7 |     :show-inheritance:
8 | 


--------------------------------------------------------------------------------
/doc/conf.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | #
  3 | # craigslist image python package documentation build configuration file
  4 | #
  5 | # All configuration values have a default; values that are commented out
  6 | # serve to show the default.
  7 | 
  8 | import os
  9 | import sys
 10 | 
 11 | # If extensions (or modules to document with autodoc) are in another directory,
 12 | # add these directories to sys.path here. If the directory is relative to the
 13 | # documentation root, use os.path.abspath to make it absolute, like shown here.
 14 | sys.path.insert(0, os.path.abspath('../climage'))
 15 | 
 16 | # -- General configuration ----------------------------------------------------
 17 | 
 18 | # Add any Sphinx extension module names here, as strings. They can be
 19 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 20 | extensions = ['sphinx.ext.autodoc']
 21 | 
 22 | # Add any paths that contain templates here, relative to this directory.
 23 | templates_path = ['_templates']
 24 | 
 25 | # The suffix of source filenames.
 26 | source_suffix = '.rst'
 27 | 
 28 | # The encoding of source files.
 29 | #source_encoding = 'utf-8'
 30 | 
 31 | # The master toctree document.
 32 | master_doc = 'index'
 33 | 
 34 | # General information about the project.
 35 | project = u'craigslist image python package'
 36 | copyright = u'2013 craigslist'
 37 | 
 38 | # The version info for the project you're documenting, acts as replacement for
 39 | # |version| and |release|, also used in various other places throughout the
 40 | # built documents.
 41 | #
 42 | # The short X.Y version.
 43 | import climage
 44 | version = 'v%s' % climage.__version__
 45 | # The full version, including alpha/beta/rc tags.
 46 | release = version
 47 | 
 48 | # The language for content autogenerated by Sphinx. Refer to documentation
 49 | # for a list of supported languages.
 50 | #language = None
 51 | 
 52 | # There are two options for replacing |today|: either, you set today to some
 53 | # non-false value, then it is used:
 54 | #today = ''
 55 | # Else, today_fmt is used as the format for a strftime call.
 56 | #today_fmt = '%B %d, %Y'
 57 | 
 58 | # List of documents that shouldn't be included in the build.
 59 | #unused_docs = []
 60 | 
 61 | # List of directories, relative to source directory, that shouldn't be searched
 62 | # for source files.
 63 | exclude_trees = ['_build']
 64 | 
 65 | # The reST default role (used for this markup: `text`) to use for documents.
 66 | #default_role = None
 67 | 
 68 | # If true, '()' will be appended to :func: etc. cross-reference text.
 69 | #add_function_parentheses = True
 70 | 
 71 | # If true, the current module name will be prepended to all description
 72 | # unit titles (such as .. function::).
 73 | #add_module_names = True
 74 | 
 75 | # If true, sectionauthor and moduleauthor directives will be shown in the
 76 | # output. They are ignored by default.
 77 | #show_authors = False
 78 | 
 79 | # The name of the Pygments (syntax highlighting) style to use.
 80 | pygments_style = 'sphinx'
 81 | 
 82 | # A list of ignored prefixes for module index sorting.
 83 | modindex_common_prefix = ['climage.']
 84 | 
 85 | 
 86 | # -- Options for HTML output --------------------------------------------------
 87 | 
 88 | # The theme to use for HTML and HTML Help pages.  Major themes that come with
 89 | # Sphinx are currently 'default' and 'sphinxdoc'.
 90 | html_theme = 'default'
 91 | 
 92 | # Theme options are theme-specific and customize the look and feel of a theme
 93 | # further.  For a list of options available for each theme, see the
 94 | # documentation.
 95 | #html_theme_options = {}
 96 | 
 97 | # Add any paths that contain custom themes here, relative to this directory.
 98 | #html_theme_path = []
 99 | 
100 | # The name for this set of Sphinx documents.  If None, it defaults to
101 | # "<project> v<release> documentation".
102 | #html_title = None
103 | 
104 | # A shorter title for the navigation bar.  Default is the same as html_title.
105 | #html_short_title = None
106 | 
107 | # The name of an image file (relative to this directory) to place at the top
108 | # of the sidebar.
109 | #html_logo = None
110 | 
111 | # The name of an image file (within the static path) to use as favicon of the
112 | # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
113 | # pixels large.
114 | #html_favicon = None
115 | 
116 | # Add any paths that contain custom static files (such as style sheets) here,
117 | # relative to this directory. They are copied after the builtin static files,
118 | # so a file named "default.css" will overwrite the builtin "default.css".
119 | #html_static_path = ['_static']
120 | 
121 | # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
122 | # using the given strftime format.
123 | #html_last_updated_fmt = '%b %d, %Y'
124 | 
125 | # If true, SmartyPants will be used to convert quotes and dashes to
126 | # typographically correct entities.
127 | #html_use_smartypants = True
128 | 
129 | # Custom sidebar templates, maps document names to template names.
130 | #html_sidebars = {}
131 | 
132 | # Additional templates that should be rendered to pages, maps page names to
133 | # template names.
134 | #html_additional_pages = {}
135 | 
136 | # If false, no module index is generated.
137 | #html_use_modindex = True
138 | 
139 | # If false, no index is generated.
140 | #html_use_index = True
141 | 
142 | # If true, the index is split into individual pages for each letter.
143 | #html_split_index = False
144 | 
145 | # If true, links to the reST sources are added to the pages.
146 | #html_show_sourcelink = True
147 | 
148 | # If true, an OpenSearch description file will be output, and all pages will
149 | # contain a <link> tag referring to it.  The value of this option must be the
150 | # base URL from which the finished HTML is served.
151 | #html_use_opensearch = ''
152 | 
153 | # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
154 | #html_file_suffix = ''
155 | 
156 | # Output file base name for HTML help builder.
157 | htmlhelp_basename = 'climagedoc'
158 | 


--------------------------------------------------------------------------------
/doc/index.rst:
--------------------------------------------------------------------------------
 1 | README
 2 | ******
 3 | 
 4 | .. include:: ../README.rst
 5 | 
 6 | Documentation
 7 | *************
 8 | 
 9 | .. automodule:: climage
10 |     :members:
11 |     :undoc-members:
12 |     :show-inheritance:
13 | 
14 | .. toctree::
15 | 
16 |     climage.exif
17 |     climage.processor
18 |     climage.server
19 | 
20 | Indices and tables
21 | ******************
22 | 
23 | * :ref:`genindex`
24 | * :ref:`modindex`
25 | * :ref:`search`
26 | 


--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [build_sphinx]
2 | all_files=1
3 | build-dir=doc/_build
4 | source-dir=doc
5 | 
6 | [nosetests]
7 | detailed-errors=1
8 | verbosity=2
9 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | # Copyright 2013 craigslist
 3 | #
 4 | # Licensed under the Apache License, Version 2.0 (the "License");
 5 | # you may not use this file except in compliance with the License.
 6 | # You may obtain a copy of the License at
 7 | #
 8 | #     http://www.apache.org/licenses/LICENSE-2.0
 9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | 
16 | '''craigslist image package setuptools script.'''
17 | 
18 | import setuptools
19 | 
20 | import climage
21 | 
22 | setuptools.setup(
23 |     name='climage',
24 |     version=climage.__version__,
25 |     description='craigslist image package',
26 |     long_description=open('README.rst').read(),
27 |     author='craigslist',
28 |     author_email='opensource@craigslist.org',
29 |     url='http://craigslist.org/about/opensource',
30 |     packages=setuptools.find_packages(exclude=['test*']),
31 |     scripts=[
32 |         'bin/climageprocessor',
33 |         'bin/climageserver'],
34 |     test_suite='nose.collector',
35 |     install_requires=[
36 |         'clblob',
37 |         'clcommon',
38 |         'pgmagick',
39 |         'PIL'],
40 |     classifiers=[
41 |         'Development Status :: 4 - Beta',
42 |         'Environment :: Console',
43 |         'Environment :: No Input/Output (Daemon)',
44 |         'Intended Audience :: Developers',
45 |         'Intended Audience :: Information Technology',
46 |         'License :: OSI Approved :: Apache Software License',
47 |         'Operating System :: POSIX :: Linux',
48 |         'Programming Language :: Python :: 2.6',
49 |         'Topic :: Software Development :: Libraries :: Python Modules'])
50 | 


--------------------------------------------------------------------------------
/test/__init__.py:
--------------------------------------------------------------------------------
 1 | # Copyright 2013 craigslist
 2 | #
 3 | # Licensed under the Apache License, Version 2.0 (the "License");
 4 | # you may not use this file except in compliance with the License.
 5 | # You may obtain a copy of the License at
 6 | #
 7 | #     http://www.apache.org/licenses/LICENSE-2.0
 8 | #
 9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | 
15 | '''Tests for craigslist image package.'''
16 | 
17 | # The worker module must be imported before any monkey patching happens.
18 | import clcommon.worker
19 | import gevent.monkey
20 | gevent.monkey.patch_all()
21 | 


--------------------------------------------------------------------------------
/test/test.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/craigslist/python-climage/377ca78fe1949d40c101c9f835ed2a1876998d2c/test/test.jpg


--------------------------------------------------------------------------------
/test/test_exif.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/craigslist/python-climage/377ca78fe1949d40c101c9f835ed2a1876998d2c/test/test_exif.jpg


--------------------------------------------------------------------------------
/test/test_processor.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2013 craigslist
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | 
 15 | '''Tests for craigslist image processor module.'''
 16 | 
 17 | import json
 18 | import PIL.Image
 19 | import os
 20 | import shutil
 21 | import StringIO
 22 | import unittest
 23 | 
 24 | import clblob.client
 25 | import clcommon.config
 26 | import clcommon.http
 27 | import climage.processor
 28 | 
 29 | IMAGE = 'test/test.jpg'
 30 | EXIF_IMAGE = 'test/test_exif.jpg'
 31 | CONFIG = clcommon.config.update(climage.processor.DEFAULT_CONFIG, {
 32 |     'clblob': {
 33 |         'client': {
 34 |             'clusters': [[{"replicas": ["test"], "write_weight": 1}]],
 35 |             'replica': 'test',
 36 |             'replicas': {
 37 |                 "test": {"ip": "127.0.0.1", "port": 12345, "read_weight": 0}}},
 38 |         'index': {
 39 |             'sqlite': {
 40 |                 'database': 'test_blob/_index',
 41 |                 'sync': 0}},
 42 |         'store': {
 43 |             'disk': {
 44 |                 'path': 'test_blob',
 45 |                 'sync': False}}}})
 46 | 
 47 | 
 48 | class TestProcessor(unittest.TestCase):
 49 | 
 50 |     config = CONFIG
 51 | 
 52 |     def __init__(self, *args, **kwargs):
 53 |         super(TestProcessor, self).__init__(*args, **kwargs)
 54 |         self.blob_servers = None
 55 | 
 56 |     def setUp(self):
 57 |         shutil.rmtree('test_blob', ignore_errors=True)
 58 |         os.makedirs('test_blob')
 59 | 
 60 |     def test_process(self):
 61 |         processor = climage.processor.Processor(self.config, open(IMAGE))
 62 |         processed = processor.process()
 63 |         self.assertEquals(processor.info['width'], 1000)
 64 |         self.assertEquals(processor.info['height'], 750)
 65 |         self.assertEquals(len(processed),
 66 |             len(self.config['climage']['processor']['sizes']))
 67 | 
 68 |     def test_convert(self):
 69 |         image = PIL.Image.open(open(IMAGE))
 70 |         output = StringIO.StringIO()
 71 |         image = image.convert(mode='LA')
 72 |         image.save(output, 'PNG')
 73 |         image = output.getvalue()
 74 |         processor = climage.processor.Processor(self.config, image)
 75 |         processed = processor.process()
 76 |         image = PIL.Image.open(StringIO.StringIO(processed['50x50c']))
 77 |         self.assertEquals(image.mode, 'RGB')
 78 | 
 79 |     def test_invalid_format(self):
 80 |         image = PIL.Image.open(open(IMAGE))
 81 |         output = StringIO.StringIO()
 82 |         image.save(output, 'PPM')
 83 |         image = output.getvalue()
 84 |         processor = climage.processor.Processor(self.config, image)
 85 |         self.assertRaises(climage.processor.BadImage, processor.process)
 86 | 
 87 |     def test_bad_file(self):
 88 |         processor = climage.processor.Processor(self.config, "bad")
 89 |         self.assertRaises(climage.processor.BadImage, processor.process)
 90 | 
 91 |     def test_too_large(self):
 92 |         config = clcommon.config.update_option(self.config,
 93 |             'climage.processor.max_width', 100)
 94 |         processor = climage.processor.Processor(config, open(IMAGE))
 95 |         self.assertRaises(climage.processor.BadImage, processor.process)
 96 | 
 97 |     def test_no_size(self):
 98 |         config = clcommon.config.update_option(self.config,
 99 |             'climage.processor.sizes', [])
100 |         processor = climage.processor.Processor(config, open(IMAGE))
101 |         processed = processor.process()
102 |         self.assertEquals(len(processed), 0)
103 | 
104 |     def test_invalid_size(self):
105 |         config = clcommon.config.update_option(self.config,
106 |             'climage.processor.sizes', 'bad size')
107 |         self.assertRaises(climage.processor.ProcessingError,
108 |             climage.processor.Processor, config, open(IMAGE))
109 | 
110 |     def test_save_blob(self):
111 |         processor = climage.processor.Processor(self.config, open(IMAGE))
112 |         images = processor.process()
113 |         self.assertTrue('blob_info_name' in processor.info)
114 |         self.assertTrue('blob_names' in processor.info)
115 |         client = clblob.client.Client(self.config)
116 |         info = processor.info.copy()
117 |         info.pop('blob_info_name')
118 |         info.pop('blob_names')
119 |         self.assertEquals(info,
120 |             json.loads(client.get(processor.info['blob_info_name']).read()))
121 |         for size in images:
122 |             self.assertEquals(images[size],
123 |                 client.get(processor.info['blob_names'][size]).read())
124 | 
125 |     def test_save_blob_fail(self):
126 |         config = clcommon.config.update_option(self.config,
127 |             'clblob.client.replica', None)
128 |         processor = climage.processor.Processor(config, open(IMAGE))
129 |         self.assertRaises(clblob.RequestError, processor.process)
130 | 
131 |     def test_truncate(self):
132 |         image = open(IMAGE).read()[:-100]
133 |         processor = climage.processor.Processor(self.config, image)
134 |         processed = processor.process()
135 |         self.assertEquals(len(processed),
136 |             len(self.config['climage']['processor']['sizes']))
137 | 
138 |     def test_pgmagick(self):
139 |         # pylint: disable=W0212
140 |         processor = climage.processor.Processor(self.config, open(IMAGE))
141 |         processor._pgmagick()
142 |         self.assertRaises(climage.processor.BadImage, processor._pgmagick)
143 | 
144 |     def test_exif(self):
145 |         processor = climage.processor.Processor(self.config, open(EXIF_IMAGE))
146 |         processor.process()
147 |         self.assertEquals(processor.info['exif_gpsversionid'], '(2, 2, 0, 0)')
148 | 
149 | 
150 | class TestProcessorNoThreads(TestProcessor):
151 | 
152 |     config = clcommon.config.update_option(CONFIG,
153 |         'climage.processor.pool_size', 0)
154 | 


--------------------------------------------------------------------------------
/test/test_server.py:
--------------------------------------------------------------------------------
  1 | # Copyright 2013 craigslist
  2 | #
  3 | # Licensed under the Apache License, Version 2.0 (the "License");
  4 | # you may not use this file except in compliance with the License.
  5 | # You may obtain a copy of the License at
  6 | #
  7 | #     http://www.apache.org/licenses/LICENSE-2.0
  8 | #
  9 | # Unless required by applicable law or agreed to in writing, software
 10 | # distributed under the License is distributed on an "AS IS" BASIS,
 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 12 | # See the License for the specific language governing permissions and
 13 | # limitations under the License.
 14 | 
 15 | '''Tests for craigslist image server module.'''
 16 | 
 17 | import httplib
 18 | import json
 19 | import os.path
 20 | import PIL.Image
 21 | import shutil
 22 | import StringIO
 23 | import unittest
 24 | 
 25 | import clcommon.config
 26 | import clcommon.http
 27 | import climage.server
 28 | import test.test_processor
 29 | 
 30 | HOST = '127.0.0.1'
 31 | PORT = 8123
 32 | CONFIG = clcommon.config.update(climage.server.DEFAULT_CONFIG, {
 33 |     'clcommon': {
 34 |         'http': {
 35 |             'host': HOST,
 36 |             'port': PORT}}})
 37 | CONFIG = clcommon.config.update(CONFIG, test.test_processor.CONFIG)
 38 | IMAGE = open(test.test_processor.IMAGE).read()
 39 | 
 40 | 
 41 | def request(method, url, *args, **kwargs):
 42 |     '''Perform the request and handle the response.'''
 43 |     connection = httplib.HTTPConnection(HOST, PORT)
 44 |     connection.request(method, url, *args, **kwargs)
 45 |     return connection.getresponse()
 46 | 
 47 | 
 48 | class TestServer(unittest.TestCase):
 49 | 
 50 |     server_class = climage.server.Server
 51 |     request_class = climage.server.Request
 52 | 
 53 |     def __init__(self, *args, **kwargs):
 54 |         super(TestServer, self).__init__(*args, **kwargs)
 55 |         self.server = None
 56 | 
 57 |     def start_server(self, config=None):
 58 |         '''Start the server, stopping an old one if needed.'''
 59 |         config = config or CONFIG
 60 |         self.stop_server()
 61 |         self.server = climage.server.Server(config, climage.server.Request)
 62 |         self.server.start()
 63 | 
 64 |     def stop_server(self):
 65 |         '''Stop the server if running.'''
 66 |         if self.server is not None:
 67 |             self.server.stop()
 68 |             self.server = None
 69 | 
 70 |     def setUp(self):
 71 |         shutil.rmtree('test_blob', ignore_errors=True)
 72 |         os.makedirs('test_blob')
 73 |         self.start_server()
 74 | 
 75 |     def tearDown(self):
 76 |         self.stop_server()
 77 | 
 78 |     def test_methods(self):
 79 |         response = request('GET', '/')
 80 |         self.assertEquals(405, response.status)
 81 |         response = request('DELETE', '/')
 82 |         self.assertEquals(405, response.status)
 83 |         response = request('PUT', '/')
 84 |         self.assertEquals(415, response.status)
 85 |         response = request('POST', '/')
 86 |         self.assertEquals(415, response.status)
 87 | 
 88 |     def test_bad_data(self):
 89 |         config = clcommon.config.update_option(CONFIG,
 90 |             'climage.server.save_bad_path', 'test_image_bad')
 91 |         self.start_server(config)
 92 |         shutil.rmtree('test_image_bad', ignore_errors=True)
 93 |         response = request('PUT', '/?filename=a/b\x00\x80/%s' % ('c' * 1000),
 94 |             'bad data')
 95 |         self.assertEquals(415, response.status)
 96 |         self.assertEquals(True, os.path.isdir('test_image_bad'))
 97 |         self.assertNotEquals(0, len(os.listdir('test_image_bad')))
 98 | 
 99 |     def test_no_sizes(self):
100 |         response = request('PUT', '/?sizes=', IMAGE)
101 |         self.assertEquals(200, response.status)
102 | 
103 |     def test_response_default(self):
104 |         response = request('PUT', '/', IMAGE)
105 |         self.assertEquals(200, response.status)
106 |         self.assertEquals(64, len(response.read()))
107 |         self.assertEquals('text/plain', response.getheader('Content-Type'))
108 | 
109 |     def test_response_none(self):
110 |         response = request('PUT', '/?response=none', IMAGE)
111 |         self.assertEquals(0, len(response.read()))
112 |         self.assertEquals(200, response.status)
113 | 
114 |     def test_response_checksum(self):
115 |         response = request('PUT', '/?response=checksum', IMAGE)
116 |         self.assertEquals(200, response.status)
117 |         self.assertEquals(64, len(response.read()))
118 |         self.assertEquals('text/plain', response.getheader('Content-Type'))
119 | 
120 |     def test_response_info(self):
121 |         response = request('PUT', '/?response=info', IMAGE)
122 |         self.assertEquals(200, response.status)
123 |         self.assertEquals('application/json',
124 |             response.getheader('Content-Type'))
125 |         info = json.loads(response.read())
126 |         self.assertEquals(64, len(info['checksum']))
127 | 
128 |     def test_response_image(self):
129 |         response = request('PUT', '/?response=50x50c', IMAGE)
130 |         self.assertEquals(200, response.status)
131 |         self.assertNotEquals(0, len(response.read()))
132 |         self.assertEquals('image/jpeg', response.getheader('Content-Type'))
133 | 
134 |     def test_response_bad(self):
135 |         response = request('PUT', '/?response=bad', IMAGE)
136 |         self.assertEquals(400, response.status)
137 | 
138 |     def test_invalid_format(self):
139 |         image = PIL.Image.open(open(test.test_processor.IMAGE))
140 |         output = StringIO.StringIO()
141 |         image.save(output, 'PPM')
142 |         response = request('PUT', '/', output.getvalue())
143 |         self.assertEquals(415, response.status)
144 | 
145 |     def test_too_large(self):
146 |         config = clcommon.config.update_option(CONFIG,
147 |             'climage.processor.max_width', 100)
148 |         self.start_server(config)
149 |         response = request('PUT', '/', IMAGE)
150 |         self.assertEquals(415, response.status)
151 | 
152 |     def test_param_sizes(self):
153 |         response = request('PUT', '/?sizes=20x20,50x50c', IMAGE)
154 |         self.assertEquals(200, response.status)
155 |         response = request('PUT', '/?sizes=', IMAGE)
156 |         self.assertEquals(200, response.status)
157 |         response = request('PUT', '/?sizes=bad', IMAGE)
158 |         self.assertEquals(400, response.status)
159 | 
160 |     def test_param_ttl(self):
161 |         response = request('PUT', '/?ttl=100', IMAGE)
162 |         self.assertEquals(200, response.status)
163 | 
164 |     def test_param_quality(self):
165 |         response = request('PUT', '/?quality=10', IMAGE)
166 |         self.assertEquals(200, response.status)
167 | 
168 |     def test_param_save(self):
169 |         response = request('PUT', '/?save=true', IMAGE)
170 |         self.assertEquals(200, response.status)
171 |         response = request('PUT', '/?save=false', IMAGE)
172 | 
173 |     def test_param_filename(self):
174 |         response = request('PUT', '/?filename=test', IMAGE)
175 |         self.assertEquals(200, response.status)
176 | 


--------------------------------------------------------------------------------