├── .gitignore ├── AUTHORS ├── LICENSE ├── MANIFEST.in ├── README.rst ├── mediasync ├── __init__.py ├── backends │ ├── __init__.py │ ├── cloudfiles.py │ ├── dummy.py │ └── s3.py ├── conf.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ └── syncmedia.py ├── models.py ├── processors │ ├── __init__.py │ ├── closurecompiler.py │ ├── slim.py │ └── yuicompressor.py ├── signals.py ├── templatetags │ ├── __init__.py │ └── media.py ├── tests │ ├── __init__.py │ ├── media │ │ ├── _test │ │ │ ├── joined.css │ │ │ └── joined.js │ │ ├── css │ │ │ ├── 1.css │ │ │ ├── 2.css │ │ │ └── 3.scss │ │ ├── img │ │ │ └── black.png │ │ └── js │ │ │ ├── 1.js │ │ │ └── 2.js │ ├── models.py │ ├── run_tests.sh │ ├── settings.py │ └── tests.py ├── urls.py └── views.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .project 3 | .pydevproject 4 | .sass-cache/ 5 | build 6 | mediasync_example 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | The following individuals have contributed to this project: 2 | 3 | * Jeremy Carbaugh 4 | * Rob Hudson 5 | * James Turk 6 | * Greg Taylor 7 | * Peter Sanchez 8 | * Jonathan Drosdeck 9 | * Rich Leland 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011, Sunlight Foundation 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | * Neither the name of Sunlight Foundation nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 9 | 10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS 2 | include LICENSE 3 | include README.rst 4 | include requirements.txt 5 | recursive-include mediasync/tests/media * -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================ 2 | django-mediasync 3 | ================ 4 | 5 | One of the more significant development roadblocks we have relates to local vs. 6 | deployed media. Ideally all media (graphics, css, scripts) development would 7 | occur locally and not use production media. Then, when ready to deploy, the 8 | media should be pushed to production. That way there can be significant changes 9 | to media without disturbing the production web site. 10 | 11 | The goal of mediasync is to develop locally and then flip a switch in production 12 | that makes all the media URLs point to remote media URLs instead of the local 13 | media directory. 14 | 15 | All code is under a BSD-style license, see LICENSE for details. 16 | 17 | Source: http://github.com/sunlightlabs/django-mediasync/ 18 | 19 | 20 | ------------ 21 | Requirements 22 | ------------ 23 | 24 | * django >= 1.0 25 | * boto >= 1.8d 26 | * slimmer == 0.1.30 (optional) 27 | * python-cloudfiles == 1.7.5 (optional, for Rackspace Cloud Files backend) 28 | 29 | ---------------------------- 30 | Upgrading from mediasync 1.x 31 | ---------------------------- 32 | 33 | #. Update your mediasync settings as described in the next section. 34 | #. Run *./manage.py syncmedia --force* to force updates of all files: 35 | * gzip instead of deflate compression 36 | * sync both compressed and original versions of files 37 | #. add "django.core.context_processors.request" to TEMPLATE_CONTEXT_PROCESSORS 38 | 39 | ------------------------------------- 40 | _`An important note about Django 1.3` 41 | ------------------------------------- 42 | 43 | When DEBUG = True and the project is run with *manage.py runserver*, Django 1.3 44 | automatically adds django.views.static.serve to urlpatterns. While this feature 45 | makes local development easier for most people, it screws everything up if 46 | you've added mediasync.urls to urlpatterns. As of now, the only way I can find 47 | to disable the automatic addition of django.views.static.serve is to use a full 48 | URL for STATIC_URL instead of just a path:: 49 | 50 | STATIC_URL = "http://localhost:8000/static/" 51 | 52 | ------------- 53 | Configuration 54 | ------------- 55 | 56 | settings.py 57 | =========== 58 | 59 | Add to *INSTALLED_APPS*:: 60 | 61 | 'mediasync' 62 | 63 | Add to *TEMPLATE_CONTEXT_PROCESSORS*:: 64 | 65 | 'django.core.context_processors.request' 66 | 67 | Make sure your *STATIC_ROOT* setting is the correct path to your media:: 68 | 69 | STATIC_ROOT = '/path/to/media' 70 | 71 | When media is being served locally (instead of from S3 or Cloud Files), 72 | mediasync serves media through a Django view. Set your *STATIC_URL* to what 73 | you'd like that local media URL to be. This can be whatever you'd like, as long 74 | as you're using the {% media_url %} tag (more details on this later):: 75 | 76 | STATIC_URL = 'http://localhost:8000/devmedia/' 77 | 78 | *STATIC_URL* is the URL that will be used in debug mode. Otherwise, 79 | the *STATIC_URL* will be loaded from the backend settings. Please see 80 | `An important note about Django 1.3`_. 81 | 82 | The following settings dict must also be added:: 83 | 84 | MEDIASYNC = { 85 | 'BACKEND': 'path.to.backend', 86 | } 87 | 88 | If you want to use a different media URL than that specified 89 | in *settings.STATIC_URL*, you can add *STATIC_URL* to the *MEDIASYNC* 90 | settings dict:: 91 | 92 | MEDIASYNC = { 93 | ... 94 | 'STATIC_URL': '/url/to/media/', # becomes http://yourhost.com/url/to/media/ 95 | ... 96 | } 97 | 98 | Same goes for *STATIC_ROOT*:: 99 | 100 | MEDIASYNC = { 101 | ... 102 | 'STATIC_ROOT': '/path/to/media/', 103 | ... 104 | } 105 | 106 | mediasync supports pluggable backends. Please see below for information on 107 | the provided backends as well as directions on implementing your own. 108 | 109 | Media expiration 110 | ---------------- 111 | 112 | If the client supports media expiration, all files are set to expire 365 days 113 | after the file was synced. You may override this value by adding 114 | *EXPIRATION_DAYS* to the MEDIASYNC settings dict. 115 | 116 | :: 117 | 118 | # Expire in 10 years. 119 | MEDIASYNC['EXPIRATION_DAYS'] = 365 * 10 120 | 121 | Serving media remote (S3/Cloud Files) or locally 122 | ------------------------------------------------ 123 | 124 | The media URL is selected based on the *SERVE_REMOTE* attribute in the 125 | *MEDIASYNC* dict in settings.py. When *False*, media will be served locally 126 | instead of from S3. 127 | 128 | :: 129 | 130 | # This would force mediasync to serve all media through the value 131 | # specified in settings.STATIC_URL. 132 | MEDIASYNC['SERVE_REMOTE'] = False 133 | 134 | # This would serve all media through S3/Cloud Files. 135 | MEDIASYNC['SERVE_REMOTE'] = True 136 | 137 | # This would serve media locally while in DEBUG mode, and remotely when 138 | # in production (DEBUG == False). 139 | MEDIASYNC['SERVE_REMOTE'] = not DEBUG 140 | 141 | When serving files locally, you can emulate the CSS/JS combo/minifying 142 | behavior we get from using media processors by specifying the following. 143 | 144 | :: 145 | 146 | MEDIASYNC['SERVE_REMOTE'] = False 147 | MEDIASYNC['EMULATE_COMBO'] = True 148 | 149 | Note that this will only work if your *STATIC_URL* is pointing at your 150 | Django dev server. Also keep in mind that some processors may take a while, 151 | and is best used to check things over before rolling out to production. 152 | 153 | DOCTYPE 154 | ------- 155 | 156 | link and script tags are written using XHTML syntax. The rendering can be 157 | overridden by using the *DOCTYPE* setting. Allowed values are *'html4'*, 158 | *'html5'*, or *'xhtml'*. The default in mediasync 2.0 is html5, just as 159 | the DOCTYPE on your site should be. 160 | 161 | :: 162 | 163 | MEDIASYNC['DOCTYPE'] = 'html5' 164 | 165 | For each doctype, the following tags are rendered: 166 | 167 | html4 168 | ~~~~~ 169 | 170 | :: 171 | 172 | 173 | 174 | 175 | html5 176 | ~~~~~ 177 | 178 | :: 179 | 180 | 181 | 182 | 183 | xhtml 184 | ~~~~~ 185 | 186 | :: 187 | 188 | 189 | 190 | 191 | 192 | SSL 193 | --- 194 | 195 | mediasync will attempt to intelligently determine if your media should be 196 | served using HTTPS. In order to use automatic SSL detection, 197 | *django.core.context_processors.request* must be added to 198 | *TEMPLATE_CONTEXT_PROCESSORS* in settings.py:: 199 | 200 | TEMPLATE_CONTEXT_PROCESSORS = ( 201 | ... 202 | 'django.core.context_processors.request', 203 | ... 204 | ) 205 | 206 | The *USE_SSL* mediasync setting can be used to override the SSL 207 | URL detection. 208 | 209 | :: 210 | 211 | # Force HTTPS. 212 | MEDIASYNC['USE_SSL'] = True 213 | 214 | or 215 | 216 | :: 217 | 218 | # Force HTTP. 219 | MEDIASYNC['USE_SSL'] = False 220 | 221 | Some backends will be unable to use SSL. In these cases *USE_SSL* and SSL 222 | detection will be ignored. 223 | 224 | urls.py 225 | ======= 226 | 227 | To serve local media through mediasync, add a reference to mediasync.urls in 228 | your main urls.py file. 229 | 230 | :: 231 | 232 | urlpatterns = ('', 233 | ... 234 | url(r'^', include('mediasync.urls)), 235 | ... 236 | ) 237 | 238 | Backends 239 | ======== 240 | 241 | mediasync now supports pluggable backends. A backend is a Python module that 242 | contains a Client class that implements a mediasync-provided BaseClient class. 243 | 244 | S3 245 | -- 246 | 247 | :: 248 | 249 | MEDIASYNC['BACKEND'] = 'mediasync.backends.s3' 250 | 251 | Settings 252 | ~~~~~~~~ 253 | 254 | The following settings are required in the mediasync settings dict:: 255 | 256 | MEDIASYNC = { 257 | 'AWS_KEY': "s3_key", 258 | 'AWS_SECRET': "s3_secret", 259 | 'AWS_BUCKET': "bucket_name", 260 | } 261 | 262 | Optionally you may specify a path prefix:: 263 | 264 | MEDIASYNC['AWS_PREFIX'] = "key_prefix" 265 | 266 | Assuming a correct DNS CNAME entry, setting *AWS_BUCKET* to 267 | *assets.sunlightlabs.com* and *AWS_PREFIX* to *myproject/media* would 268 | sync the media directory to http://assets.sunlightlabs.com/myproject/media/. 269 | 270 | Amazon allows users to create DNS CNAME entries to map custom domain names 271 | to an AWS bucket. MEDIASYNC can be configured to use the bucket as the media 272 | URL by setting *AWS_BUCKET_CNAME* to *True*. 273 | 274 | :: 275 | 276 | MEDIASYNC['AWS_BUCKET_CNAME'] = True 277 | 278 | If you would prefer to not use gzip compression with the S3 client, it can be 279 | disabled:: 280 | 281 | MEDIASYNC['AWS_GZIP'] = False 282 | 283 | Tips 284 | ~~~~ 285 | 286 | Since files are given a far future expires header, one needs a way to do 287 | "cache busting" when you want the browser to fetch new files before the expire 288 | date arrives. One of the best and easiest ways to accomplish this is to alter 289 | the path to the media files with some sort of version string using the key 290 | prefix setting:: 291 | 292 | MEDIASYNC['AWS_PREFIX'] = "myproject/media/v20001201" 293 | 294 | Given that and the above DNS CNAME example, the media directory URL would end 295 | up being http://assets.sunlightlabs.com/myproject/media/v20001201/. Whenever 296 | you need to update the media files, simply update the key prefix with a new 297 | versioned string. 298 | 299 | A *CACHE_BUSTER* settings can be added to the main *MEDIASYNC* settings 300 | dict to add a query string parameter to all media URLs. The cache buster can 301 | either be a value or a callable which is passed the media URL as a parameter. 302 | 303 | :: 304 | 305 | MEDIASYNC['CACHE_BUSTER'] = 1234567890 306 | 307 | The above setting will generate a media path similar to:: 308 | 309 | http://yourhost.com/url/to/media/image.png?1234567890 310 | 311 | An important thing to note is that if you're running your Django site in a 312 | multi-threaded or multi-node setup, you'll want to be careful about using a 313 | time-based cache buster value. Each worker/thread will probably have a slightly 314 | different value for datetime.now(), which means your users will find themselves 315 | having cache misses randomly from page to page. 316 | 317 | Rackspace Cloud Files 318 | --------------------- 319 | 320 | :: 321 | 322 | MEDIASYNC['BACKEND'] = 'mediasync.backends.cloudfiles' 323 | 324 | Settings 325 | ~~~~~~~~ 326 | 327 | The following settings are required in the mediasync settings dict:: 328 | 329 | MEDIASYNC = { 330 | 'CLOUDFILES_CONTAINER': 'container_name', 331 | 'CLOUDFILES_USERNAME': 'cf_username', 332 | 'CLOUDFILES_API_KEY': 'cf_apikey', 333 | } 334 | 335 | Tips 336 | ~~~~ 337 | 338 | The Cloud Files backend lacks support for the following features: 339 | 340 | * setting HTTP Expires header 341 | * setting HTTP Cache-Control header 342 | * content compression (gzip) 343 | * SSL support 344 | * conditional sync based on file checksum 345 | 346 | Custom backends 347 | --------------- 348 | 349 | You can create a custom backend by creating a Python module containing a Client 350 | class. This class must inherit from mediasync.backends.BaseClient. Additionally, 351 | you must implement two methods:: 352 | 353 | def remote_media_url(self, with_ssl): 354 | ... 355 | 356 | *remote_media_url* returns the full base URL for remote media. This can be 357 | either a static URL or one generated from mediasync settings:: 358 | 359 | def put(self, filedata, content_type, remote_path, force=False): 360 | ... 361 | 362 | put is responsible for pushing a file to the backend storage. 363 | 364 | * filedata - the contents of the file 365 | * content_type - the mime type of the file 366 | * remote_path - the remote path (relative from remote_media_url) to which 367 | the file should be written 368 | * force - if True, write file to remote storage even if it already exists 369 | 370 | If the client supports gzipped content, you will need to override supports_gzip 371 | to return True:: 372 | 373 | def supports_gzip(self): 374 | return True 375 | 376 | File Processors 377 | =============== 378 | 379 | File processors allow you to modify the content of a file as it is being 380 | synced or served statically. 381 | 382 | Mediasync ships with three processor modules: 383 | 384 | 1. ``slim`` is a minifier written in Python and requires the 385 | `slimmer` Python package. The Python package can be found here: 386 | http://pypi.python.org/pypi/slimmer/ 387 | 388 | 2. ``yuicompressor`` is a minifier written in Java and can be downloaded 389 | from YUI's download page: http://developer.yahoo.com/yui/compressor/. 390 | This processor also requires an additional setting, as defined below. 391 | `yuicompressor` is new and should be considered experimental until 392 | the mediasync 2.1 release. 393 | 394 | 3. ``closurecompiler`` is a javascript compiler provided by Google. 395 | 396 | Custom processors can be specified using the *PROCESSORS* entry in the 397 | mediasync settings dict. *PROCESSORS* should be a list of processor entries. 398 | Each processor entry can be a callable or a string path to a callable. If the 399 | path is to a class definition, the class will be instantiated into an object. 400 | The processor callable should return a string of the processed file data, None 401 | if it chooses to not process the file, or raise *mediasync.SyncException* if 402 | something goes terribly wrong. The callable should take the following arguments:: 403 | 404 | def proc(filedata, content_type, remote_path, is_active): 405 | ... 406 | 407 | filedata 408 | the content of the file as a string 409 | 410 | content_type 411 | the mimetype of the file being processed 412 | 413 | remote_path 414 | the path to which the file is being synced (contains the file name) 415 | 416 | is_active 417 | True if the processor should... process 418 | 419 | If the *PROCESSORS* setting is used, you will need to include the defaults 420 | if you plan on using them:: 421 | 422 | 'PROCESSORS': ( 423 | 'mediasync.processors.slim.css_minifier', 424 | 'mediasync.processors.slim.js_minifier', 425 | ... 426 | ), 427 | 428 | mediasync will attempt to use `slimmer` by default if you have the package 429 | installed and do not use the PROCESSORS setting. 430 | 431 | Google Closure Compiler 432 | ----------------------- 433 | 434 | Google's JavaScript Closure Compiler provides an API that allows files to be 435 | compressed without installing anything locally. To use the service:: 436 | 437 | 'PROCESSORS': ('mediasync.processors.closurecompiler.compile',) 438 | 439 | YUI Compressor 440 | -------------- 441 | 442 | To configure YUI Compressor you need to define a `PROCESSORS` and 443 | `YUI_COMPRESSOR_PATH` as follows, assuming you placed the ".jar" file in 444 | your `~/bin` path:: 445 | 446 | 'PROCESSORS': ('mediasync.processors.yuicompressor.css_minifier', 447 | 'mediasync.processors.yuicompressor.js_minifier'), 448 | 'YUI_COMPRESSOR_PATH': '~/bin/yuicompressor.jar', 449 | 450 | -------- 451 | Features 452 | -------- 453 | 454 | Ignored Directories 455 | =================== 456 | 457 | Any directory in *STATIC_ROOT* that is hidden or starts with an underscore 458 | will be ignored during syncing. 459 | 460 | 461 | Template Tags 462 | ============= 463 | 464 | When referring to media in HTML templates you can use custom template tags. 465 | These tags can by accessed by loading the media template tag collection. 466 | 467 | :: 468 | 469 | {% load media %} 470 | 471 | Any tag that has a path argument can use either a string or a variable:: 472 | 473 | {% media_url "images/avatar.png" } 474 | {% media_url user.profile.avatar_path %} 475 | 476 | Some backends (S3) support https URLs when the requesting page is secure. 477 | In order for the https to be detected, the request must be placed in the 478 | template context with the key 'request'. This can be done automatically by 479 | adding 'django.core.context_processors.request' to *TEMPLATE_CONTEXT_PROCESSORS* 480 | in settings.py 481 | 482 | media_url 483 | --------- 484 | 485 | Renders the STATIC_URL from settings.py with trailing slashes removed. 486 | 487 | :: 488 | 489 | 490 | 491 | STATIC_URL takes an optional argument that is the media path. Using the argument 492 | allows mediasync to add the CACHE_BUSTER to the URL if one is specified. 493 | 494 | :: 495 | 496 | 497 | 498 | If *CACHE_BUSTER* is set to 12345, the above example will render as:: 499 | 500 | 501 | 502 | *NOTE*: Don't use this tag to serve CSS or JS files. Use the js and css tags 503 | that were specifically designed for the purpose. 504 | 505 | 506 | js 507 | -- 508 | 509 | Renders a script tag with the correct include. 510 | 511 | :: 512 | 513 | {% js "myfile.js" %} 514 | 515 | 516 | css 517 | --- 518 | 519 | Renders a tag to include the stylesheet. It takes an optional second 520 | parameter for the media attribute; the default media is "screen, projector". 521 | 522 | :: 523 | 524 | {% css "myfile.css" %} 525 | {% css "myfile.css" "screen" %} 526 | 527 | 528 | css_print 529 | --------- 530 | 531 | Shortcut to render as a print stylesheet. 532 | 533 | :: 534 | 535 | {% css_print "myfile.css" %} 536 | 537 | which is equivalent to 538 | 539 | :: 540 | 541 | {% css "myfile.css" "print" %} 542 | 543 | Writing Style Sheets 544 | ==================== 545 | 546 | Users are encouraged to write stylesheets using relative URLS. The media 547 | directory is synced with S3 as is, so relative local paths will still work 548 | when pushed remotely. 549 | 550 | :: 551 | 552 | background: url(../images/arrow_left.png); 553 | 554 | 555 | Joined files 556 | ============ 557 | 558 | When serving media in production, it is beneficial to combine JavaScript and 559 | CSS into single files. This reduces the number of connections the browser needs 560 | to make to the web server. Fewer connections can dramatically decrease page 561 | load times and reduce the server-side load. 562 | 563 | Joined files are specified in the *MEDIASYNC* dict using *JOINED*. This is 564 | a dict that maps individual media to an alias for the joined files. 565 | 566 | :: 567 | 568 | 'JOINED': { 569 | 'styles/joined.css': ['styles/reset.css','styles/text.css'], 570 | 'scripts/joined.js': ['scripts/jquery.js','scripts/processing.js'], 571 | }, 572 | 573 | Files listed in *JOINED* will be combined and pushed to S3 with the name of 574 | the alias. The individual CSS files will also be pushed to S3. Aliases must end 575 | in either .css or .js in order for the content-type to be set appropriately. 576 | 577 | The existing template tags may be used to refer to the joined media. Simply use 578 | the joined alias as the argument:: 579 | 580 | {% css_print "joined.css" %} 581 | 582 | When served locally, template tags will render an HTML tag for each of the files 583 | that make up the joined file:: 584 | 585 | 586 | 587 | 588 | When served remotely, one HTML tag will be rendered with the name of the joined file:: 589 | 590 | 591 | 592 | Smart GZIP for S3 593 | ================= 594 | 595 | In previous versions of mediasync's S3 client, certain content was always pushed 596 | in a compressed format. This can cause major issues with clients that do not 597 | support gzip. New in version 2.0, mediasync will push both a gzipped and an 598 | uncompressed version of the file to S3. The template tags look at the request 599 | and direct the user to the appropriate file based on the ACCEPT_ENCODING HTTP 600 | header. Assuming a file styles/layout.css, the following would be synced to S3:: 601 | 602 | styles/layout.css 603 | styles/layout.css.gzt 604 | 605 | Note the altered use of the .gz extension. Some versions of the Safari browser 606 | on OSX ignore the Content-Type header for files ending in .gz and treat them 607 | instead as files to download. This altered extension allows Safari to deflate 608 | and utilize the files correctly without affecting functionality in any other 609 | tested browsers. 610 | 611 | Signals 612 | ======= 613 | 614 | mediasync provides two signals that allow you to hook into the syncing 615 | process. *pre_sync* is sent after the client is opened, but before the first 616 | file is synced. *post_sync* is sent after the last file is synced, but before 617 | the client is closed. This allows you to call commands on the client without 618 | having to worry about its state. The signals allow you to do common tasks such 619 | as calling Django 1.3's collectstatic command, process SASS stylesheets, or 620 | clean up files generated during a pre_sync process. 621 | 622 | collectstatic receiver 623 | ---------------------- 624 | 625 | A receiver for calling the collectstatic management command is provided:: 626 | 627 | from mediasync.signals import pre_sync, collectstatic_receiver 628 | 629 | # run collectstatic before syncing media 630 | pre_sync.connect(collectstatic_receiver) 631 | 632 | SASS receiver 633 | ------------- 634 | 635 | A receiver for compiling SASS into CSS is provided:: 636 | 637 | from mediasync.signals import pre_sync, sass_receiver 638 | 639 | # compile SASS files before syncing media 640 | pre_sync.connect(sass_receiver) 641 | 642 | Any file in static root that has the *sass* or *scss* file extension will be 643 | compiled into CSS. The compiled CSS file will be placed in the same directory 644 | and the original extension will be replaced with *css*. If a file exists with 645 | the same *css* extension, it will be overwritten. 646 | 647 | By default mediasync uses the *sass* command with no options. If you would 648 | like to specify your own command, specify *SASS_COMMAND* in settings:: 649 | 650 | MEDIASYNC = { 651 | ... 652 | 'SASS_COMMAND': 'sass -scss -l', 653 | ... 654 | } 655 | 656 | ----------------- 657 | Running MEDIASYNC 658 | ----------------- 659 | 660 | :: 661 | 662 | ./manage.py syncmedia 663 | 664 | ---------- 665 | Change Log 666 | ---------- 667 | 668 | 2.2.0 669 | ====================== 670 | 671 | * added pre_sync and post_sync signals 672 | * provide basic receiver for calling collectstatic before syncing 673 | * provide receiver for compiling SASS before syncing 674 | * show media directory listing when serving locally in debug mode 675 | * add processor for Google's Closure Compiler API for JavaScript 676 | * template tags can now take a variable as the path argument 677 | 678 | 2.1.0 679 | ===== 680 | 681 | * default to using STATIC_URL and STATIC_ROOT (Django 1.3), falling back 682 | to MEDIA_URL and MEDIA_ROOT if the STATIC_* settings are not set 683 | * add AWS_GZIP setting to optionally disable gzip compression in S3 client 684 | 685 | Thanks to Rob Hudson and Dolan Antenucci for their contributions to this 686 | release. 687 | 688 | 2.0.0 689 | ===== 690 | 691 | * updated Rackspace Cloud Files backend 692 | * use gzip instead of deflate for compression (better browser support) 693 | * smart gzip client support detection 694 | * add pluggable backends 695 | * add pluggable file processors 696 | * experimental YUI Compressor 697 | * settings refactor 698 | * allow override of *settings.MEDIA_URL* 699 | * Improvements to the logic that decides which files to sync. Safely ignore 700 | a wider variety of hidden files/directories. 701 | * Make template tags aware of whether the current page is SSL-secured. If it 702 | is, ask the backend for an SSL media URL (if implemented by your backend). 703 | * made SERVE_REMOTE setting the sole factor in determining if 704 | media should be served locally or remotely 705 | * add many more tests 706 | * deprecate CSS_PATH and JS_PATH 707 | 708 | Thanks to Greg Taylor, Peter Sanchez, Jonathan Drosdeck, Rich Leland, 709 | and Rob Hudson for their contributions to this release. 710 | 711 | 1.0.1 712 | ===== 713 | 714 | * add application/javascript and application/x-javascript to JavaScript 715 | mimetypes 716 | * break out of CSS and JS mimetypes 717 | * add support for HTTPS URLs to S3 718 | * allow for storage of S3 keys in ~/.boto configuration file 719 | 720 | Thanks to Rob Hudson and Peter Sanchez for their contributions. 721 | 722 | 1.0.0 723 | ===== 724 | 725 | Initial release. 726 | -------------------------------------------------------------------------------- /mediasync/__init__.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import cStringIO 3 | import gzip 4 | import hashlib 5 | import mimetypes 6 | import os 7 | 8 | JS_MIMETYPES = ( 9 | "application/javascript", 10 | "application/x-javascript", 11 | "text/javascript", # obsolete, see RFC 4329 12 | ) 13 | CSS_MIMETYPES = ( 14 | "text/css", 15 | ) 16 | TYPES_TO_COMPRESS = ( 17 | "application/json", 18 | "application/xml", 19 | "text/html", 20 | "text/plain", 21 | "text/xml", 22 | ) + JS_MIMETYPES + CSS_MIMETYPES 23 | 24 | class SyncException(Exception): 25 | pass 26 | 27 | def checksum(data): 28 | checksum = hashlib.md5(data) 29 | hexdigest = checksum.hexdigest() 30 | b64digest = base64.b64encode(checksum.digest()) 31 | return (hexdigest, b64digest) 32 | 33 | def compress(s): 34 | zbuf = cStringIO.StringIO() 35 | zfile = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=zbuf) 36 | zfile.write(s) 37 | zfile.close() 38 | return zbuf.getvalue() 39 | 40 | def is_syncable_dir(dir_str): 41 | return not dir_str.startswith('.') and not dir_str.startswith('_') 42 | 43 | def is_syncable_file(file_str): 44 | return not file_str.startswith('.') and not file_str.startswith('_') 45 | 46 | def listdir_recursive(dir_str): 47 | for root, dirs, files in os.walk(dir_str): 48 | # Go through and yank any directories that don't pass our syncable 49 | # dir test. This needs to be done in place so that walk() will avoid. 50 | for dir_candidate in dirs: 51 | if not is_syncable_dir(dir_candidate): 52 | dirs.remove(dir_candidate) 53 | 54 | basename = os.path.basename(root) 55 | if is_syncable_dir(basename): 56 | for file in files: 57 | fname = os.path.join(root, file).replace(dir_str, '', 1) 58 | if fname.startswith('/'): 59 | fname = fname[1:] 60 | yield fname 61 | else: 62 | # "Skipping directory %s" % root 63 | pass 64 | 65 | def combine_files(joinfile, sourcefiles, client): 66 | """ 67 | Given a combo file name (joinfile), combine the sourcefiles into a single 68 | monolithic file. 69 | 70 | Returns a string containing the combo file, or None if the specified 71 | file can not be combo'd. 72 | """ 73 | from mediasync.conf import msettings 74 | 75 | joinfile = joinfile.strip('/') 76 | 77 | if joinfile.endswith('.css'): 78 | dirname = msettings['CSS_PATH'].strip('/') 79 | separator = '\n' 80 | elif joinfile.endswith('.js'): 81 | dirname = msettings['JS_PATH'].strip('/') 82 | separator = ';\n' 83 | else: 84 | # By-pass this file since we only join CSS and JS. 85 | return None 86 | 87 | buffer = cStringIO.StringIO() 88 | 89 | for sourcefile in sourcefiles: 90 | sourcepath = os.path.join(client.media_root, dirname, sourcefile) 91 | if os.path.isfile(sourcepath): 92 | f = open(sourcepath) 93 | buffer.write(f.read()) 94 | f.close() 95 | buffer.write(separator) 96 | 97 | filedata = buffer.getvalue() 98 | buffer.close() 99 | return (filedata, dirname) 100 | 101 | def sync(client=None, force=False, verbose=True): 102 | """ Let's face it... pushing this stuff to S3 is messy. 103 | A lot of different things need to be calculated for each file 104 | and they have to be in a certain order as some variables rely 105 | on others. 106 | """ 107 | from mediasync import backends 108 | from mediasync.conf import msettings 109 | from mediasync.signals import pre_sync, post_sync 110 | 111 | # create client connection 112 | if client is None: 113 | client = backends.client() 114 | 115 | client.open() 116 | client.serve_remote = True 117 | 118 | # send pre-sync signal 119 | pre_sync.send(sender=client) 120 | 121 | # 122 | # sync joined media 123 | # 124 | 125 | for joinfile, sourcefiles in msettings['JOINED'].iteritems(): 126 | 127 | filedata = combine_files(joinfile, sourcefiles, client) 128 | if filedata is None: 129 | # combine_files() is only interested in CSS/JS files. 130 | continue 131 | filedata, dirname = filedata 132 | 133 | content_type = mimetypes.guess_type(joinfile)[0] or msettings['DEFAULT_MIMETYPE'] 134 | 135 | remote_path = joinfile 136 | if dirname: 137 | remote_path = "%s/%s" % (dirname, remote_path) 138 | 139 | if client.process_and_put(filedata, content_type, remote_path, force=force): 140 | if verbose: 141 | print "[%s] %s" % (content_type, remote_path) 142 | 143 | # 144 | # sync static media 145 | # 146 | 147 | for dirname in os.listdir(client.media_root): 148 | 149 | dirpath = os.path.abspath(os.path.join(client.media_root, dirname)) 150 | 151 | if os.path.isdir(dirpath): 152 | 153 | for filename in listdir_recursive(dirpath): 154 | 155 | # calculate local and remote paths 156 | filepath = os.path.join(dirpath, filename) 157 | remote_path = "%s/%s" % (dirname, filename) 158 | 159 | content_type = mimetypes.guess_type(filepath)[0] or msettings['DEFAULT_MIMETYPE'] 160 | 161 | if not is_syncable_file(os.path.basename(filename)) or not os.path.isfile(filepath): 162 | continue # hidden file or directory, do not upload 163 | 164 | filedata = open(filepath, 'rb').read() 165 | 166 | if client.process_and_put(filedata, content_type, remote_path, force=force): 167 | if verbose: 168 | print "[%s] %s" % (content_type, remote_path) 169 | 170 | # send post-sync signal while client is still open 171 | post_sync.send(sender=client) 172 | 173 | client.close() 174 | 175 | 176 | __all__ = ['sync', 'SyncException'] 177 | __version__ = '2.2.1' 178 | -------------------------------------------------------------------------------- /mediasync/backends/__init__.py: -------------------------------------------------------------------------------- 1 | from django.core.exceptions import ImproperlyConfigured 2 | from django.utils.importlib import import_module 3 | from mediasync.conf import msettings 4 | from urlparse import urlparse 5 | 6 | def client(): 7 | backend_name = msettings['BACKEND'] 8 | if not backend_name: 9 | raise ImproperlyConfigured('must define a mediasync BACKEND property') 10 | return load_backend(backend_name) 11 | 12 | def load_backend(backend_name): 13 | try: 14 | backend = import_module(backend_name) 15 | return backend.Client() 16 | except ImportError, e: 17 | raise ImproperlyConfigured(("%s is not a valid mediasync backend. \n" + 18 | "Error was: %s") % (backend_name, e)) 19 | 20 | class BaseClient(object): 21 | 22 | def __init__(self, *args, **kwargs): 23 | 24 | # mediasync settings 25 | self.expiration_days = msettings['EXPIRATION_DAYS'] 26 | self.serve_remote = msettings['SERVE_REMOTE'] 27 | 28 | self.local_media_url = self.get_local_media_url() 29 | self.media_root = self.get_media_root() 30 | 31 | self.processors = [] 32 | for proc in msettings['PROCESSORS']: 33 | 34 | if isinstance(proc, basestring): 35 | try: 36 | dot = proc.rindex('.') 37 | except ValueError: 38 | raise exceptions.ImproperlyConfigured, '%s isn\'t a processor module' % (proc,) 39 | module, attr = proc[:dot], proc[dot+1:] 40 | module = import_module(module) 41 | proc = getattr(module, attr, None) 42 | 43 | if isinstance(proc, type): 44 | proc = proc() 45 | 46 | if callable(proc): 47 | self.processors.append(proc) 48 | 49 | def supports_gzip(self): 50 | return False 51 | 52 | def get_local_media_url(self): 53 | """ 54 | Checks msettings['STATIC_URL'], then settings.STATIC_URL. 55 | 56 | Broken out to allow overriding if need be. 57 | """ 58 | url = msettings['STATIC_URL'] 59 | return urlparse(url).path 60 | 61 | def get_media_root(self): 62 | """ 63 | Checks msettings['STATIC_ROOT'], then settings.STATIC_ROOT. 64 | 65 | Broken out to allow overriding if need be. 66 | """ 67 | return msettings['STATIC_ROOT'] 68 | 69 | def media_url(self, with_ssl=False): 70 | """ 71 | Used to return a base media URL. Depending on whether we're serving 72 | media remotely or locally, this either hands the decision off to the 73 | backend, or just uses the value in settings.STATIC_URL. 74 | 75 | args: 76 | with_ssl: (bool) If True, return an HTTPS url (depending on how 77 | the backend handles it). 78 | """ 79 | if self.serve_remote: 80 | # Hand this off to whichever backend is being used. 81 | url = self.remote_media_url(with_ssl) 82 | else: 83 | # Serving locally, just use the value in settings.py. 84 | url = self.local_media_url 85 | return url.rstrip('/') 86 | 87 | def process(self, filedata, content_type, remote_path): 88 | for proc in self.processors: 89 | is_active = msettings['SERVE_REMOTE'] or msettings['EMULATE_COMBO'] 90 | prcssd_filedata = proc(filedata, content_type, remote_path, is_active) 91 | if prcssd_filedata is not None: 92 | filedata = prcssd_filedata 93 | return filedata 94 | 95 | def process_and_put(self, filedata, content_type, remote_path, force=False): 96 | filedata = self.process(filedata, content_type, remote_path) 97 | return self.put(filedata, content_type, remote_path, force) 98 | 99 | def put(self, filedata, content_type, remote_path, force=False): 100 | raise NotImplementedError('put not defined in ' + self.__class__.__name__) 101 | 102 | def remote_media_url(self, with_ssl=False): 103 | raise NotImplementedError('remote_media_url not defined in ' + self.__class__.__name__) 104 | 105 | def open(self): 106 | pass 107 | 108 | def close(self): 109 | pass 110 | -------------------------------------------------------------------------------- /mediasync/backends/cloudfiles.py: -------------------------------------------------------------------------------- 1 | import cloudfiles 2 | 3 | from django.core.exceptions import ImproperlyConfigured 4 | 5 | from mediasync.backends import BaseClient 6 | from mediasync.conf import msettings 7 | 8 | 9 | class Client(BaseClient): 10 | 11 | def __init__(self, *args, **kwargs): 12 | "Set up the CloudFiles connection and grab the container." 13 | super(Client, self).__init__(*args, **kwargs) 14 | 15 | container_name = msettings['CLOUDFILES_CONTAINER'] 16 | username = msettings['CLOUDFILES_USERNAME'] 17 | key = msettings['CLOUDFILES_API_KEY'] 18 | 19 | if not container_name: 20 | raise ImproperlyConfigured("CLOUDFILES_CONTAINER is a required setting.") 21 | 22 | if not username: 23 | raise ImproperlyConfigured("CLOUDFILES_USERNAME is a required setting.") 24 | 25 | if not key: 26 | raise ImproperlyConfigured("CLOUDFILES_API_KEY is a required setting.") 27 | 28 | self.conn = cloudfiles.get_connection(username, key) 29 | self.container = self.conn.create_container(container_name) 30 | 31 | if not self.container.is_public(): 32 | self.container.make_public() 33 | 34 | def remote_media_url(self, with_ssl=False): 35 | "Grab the remote URL for the contianer." 36 | if with_ssl: 37 | raise UserWarning("""Rackspace CloudFiles does not yet support SSL. 38 | See http://bit.ly/hYV502 for more info.""") 39 | return self.container.public_uri() 40 | 41 | def put(self, filedata, content_type, remote_path, force=False): 42 | 43 | obj = self.container.create_object(remote_path) 44 | obj.content_type = content_type 45 | obj.write(filedata) 46 | 47 | return True 48 | -------------------------------------------------------------------------------- /mediasync/backends/dummy.py: -------------------------------------------------------------------------------- 1 | from mediasync.backends import BaseClient 2 | 3 | class Client(BaseClient): 4 | 5 | remote_media_url_callback = lambda x: "dummy://" 6 | put_callback = lambda x: None 7 | 8 | def remote_media_url(self, with_ssl=False): 9 | return self.remote_media_url_callback() 10 | 11 | def put(self, *args, **kwargs): 12 | self.put_callback(*args) -------------------------------------------------------------------------------- /mediasync/backends/s3.py: -------------------------------------------------------------------------------- 1 | from boto.s3.connection import S3Connection 2 | from boto.s3.key import Key 3 | from django.core.exceptions import ImproperlyConfigured 4 | from mediasync import TYPES_TO_COMPRESS 5 | from mediasync.backends import BaseClient 6 | from mediasync.conf import msettings 7 | import mediasync 8 | import datetime 9 | 10 | class Client(BaseClient): 11 | 12 | def __init__(self, *args, **kwargs): 13 | super(Client, self).__init__(*args, **kwargs) 14 | 15 | self.aws_bucket = msettings['AWS_BUCKET'] 16 | self.aws_prefix = msettings.get('AWS_PREFIX', '').strip('/') 17 | self.aws_bucket_cname = msettings.get('AWS_BUCKET_CNAME', False) 18 | 19 | assert self.aws_bucket 20 | 21 | def supports_gzip(self): 22 | return msettings.get('AWS_GZIP', True) 23 | 24 | def get_connection(self): 25 | return self._conn 26 | 27 | def open(self): 28 | 29 | key = msettings['AWS_KEY'] 30 | secret = msettings['AWS_SECRET'] 31 | 32 | try: 33 | self._conn = S3Connection(key, secret) 34 | except AttributeError: 35 | raise ImproperlyConfigured("S3 keys not set and no boto config found.") 36 | 37 | self._bucket = self._conn.create_bucket(self.aws_bucket) 38 | 39 | def close(self): 40 | self._bucket = None 41 | self._conn = None 42 | 43 | def remote_media_url(self, with_ssl=False): 44 | """ 45 | Returns the base remote media URL. In this case, we can safely make 46 | some assumptions on the URL string based on bucket names, and having 47 | public ACL on. 48 | 49 | args: 50 | with_ssl: (bool) If True, return an HTTPS url. 51 | """ 52 | protocol = 'http' if with_ssl is False else 'https' 53 | url = (self.aws_bucket_cname and "%s://%s" or "%s://s3.amazonaws.com/%s") % (protocol, self.aws_bucket) 54 | if self.aws_prefix: 55 | url = "%s/%s" % (url, self.aws_prefix) 56 | return url 57 | 58 | def put(self, filedata, content_type, remote_path, force=False): 59 | 60 | now = datetime.datetime.utcnow() 61 | then = now + datetime.timedelta(self.expiration_days) 62 | expires = then.strftime("%a, %d %b %Y %H:%M:%S GMT") 63 | 64 | if self.aws_prefix: 65 | remote_path = "%s/%s" % (self.aws_prefix, remote_path) 66 | 67 | (hexdigest, b64digest) = mediasync.checksum(filedata) 68 | raw_b64digest = b64digest # store raw b64digest to add as file metadata 69 | 70 | # create initial set of headers 71 | headers = { 72 | "x-amz-acl": "public-read", 73 | "Content-Type": content_type, 74 | "Expires": expires, 75 | "Cache-Control": 'max-age=%d, public' % (self.expiration_days * 24 * 3600), 76 | } 77 | 78 | key = self._bucket.get_key(remote_path) 79 | 80 | if key is None: 81 | key = Key(self._bucket, remote_path) 82 | 83 | key_meta = key.get_metadata('mediasync-checksum') or '' 84 | s3_checksum = key_meta.replace(' ', '+') 85 | if force or s3_checksum != raw_b64digest: 86 | 87 | key.set_metadata('mediasync-checksum', raw_b64digest) 88 | key.set_contents_from_string(filedata, headers=headers, md5=(hexdigest, b64digest)) 89 | 90 | # check to see if file should be gzipped based on content_type 91 | # also check to see if filesize is greater than 1kb 92 | if content_type in TYPES_TO_COMPRESS: 93 | # Use a .gzt extension to avoid issues with Safari on OSX 94 | key = Key(self._bucket, "%s.gzt" % remote_path) 95 | 96 | filedata = mediasync.compress(filedata) 97 | (hexdigest, b64digest) = mediasync.checksum(filedata) # update checksum with compressed data 98 | headers["Content-Disposition"] = 'inline; filename="%sgzt"' % remote_path.split('/')[-1] 99 | headers["Content-Encoding"] = 'gzip' 100 | 101 | key.set_metadata('mediasync-checksum', raw_b64digest) 102 | key.set_contents_from_string(filedata, headers=headers, md5=(hexdigest, b64digest)) 103 | 104 | return True 105 | -------------------------------------------------------------------------------- /mediasync/conf.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from mediasync.processors import slim 3 | 4 | _settings = { 5 | 'CSS_PATH': '', 6 | 'DEFAULT_MIMETYPE': 'application/octet-stream', 7 | 'DOCTYPE': 'html5', 8 | 'EMULATE_COMBO': False, 9 | 'EXPIRATION_DAYS': 365, 10 | 'JOINED': {}, 11 | 'JS_PATH': '', 12 | 'STATIC_ROOT': getattr(settings, 'STATIC_ROOT', None) or 13 | getattr(settings, 'MEDIA_ROOT', None), 14 | 'STATIC_URL': getattr(settings, 'STATIC_URL', None) or 15 | getattr(settings, 'MEDIA_URL', None), 16 | 'PROCESSORS': (slim.css_minifier, slim.js_minifier), 17 | 'SERVE_REMOTE': not settings.DEBUG, 18 | 'URL_PROCESSOR': lambda x: x, 19 | } 20 | 21 | class Settings(object): 22 | 23 | def __init__(self, conf): 24 | for k, v in conf.iteritems(): 25 | self[k] = v 26 | 27 | def __delitem__(self, name): 28 | del _settings[name] 29 | 30 | def __getitem__(self, name): 31 | return self.get(name) 32 | 33 | def __setitem__(self, name, val): 34 | _settings[name] = val 35 | 36 | def __str__(self): 37 | return repr(_settings) 38 | 39 | def get(self, name, default=None): 40 | return _settings.get(name, default) 41 | 42 | msettings = Settings(settings.MEDIASYNC) 43 | -------------------------------------------------------------------------------- /mediasync/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunlightlabs/django-mediasync/aa8ce4cfff757bbdb488463c64c0863cca6a1932/mediasync/management/__init__.py -------------------------------------------------------------------------------- /mediasync/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunlightlabs/django-mediasync/aa8ce4cfff757bbdb488463c64c0863cca6a1932/mediasync/management/commands/__init__.py -------------------------------------------------------------------------------- /mediasync/management/commands/syncmedia.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | from optparse import make_option 3 | from mediasync.conf import msettings 4 | import mediasync 5 | 6 | class Command(BaseCommand): 7 | 8 | help = "Sync local media with remote client" 9 | args = '[options]' 10 | 11 | requires_model_validation = False 12 | 13 | option_list = BaseCommand.option_list + ( 14 | make_option("-F", "--force", dest="force", help="force files to sync", action="store_true"), 15 | ) 16 | 17 | def handle(self, *args, **options): 18 | 19 | msettings['SERVE_REMOTE'] = True 20 | 21 | force = options.get('force') or False 22 | 23 | try: 24 | mediasync.sync(force=force) 25 | except ValueError, ve: 26 | raise CommandError('%s\nUsage is mediasync %s' % (ve.message, self.args)) -------------------------------------------------------------------------------- /mediasync/models.py: -------------------------------------------------------------------------------- 1 | # no models! -------------------------------------------------------------------------------- /mediasync/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunlightlabs/django-mediasync/aa8ce4cfff757bbdb488463c64c0863cca6a1932/mediasync/processors/__init__.py -------------------------------------------------------------------------------- /mediasync/processors/closurecompiler.py: -------------------------------------------------------------------------------- 1 | from mediasync import JS_MIMETYPES 2 | from urllib import urlencode 3 | import httplib 4 | 5 | HEADERS = {"content-type": "application/x-www-form-urlencoded"} 6 | 7 | def compile(filedata, content_type, remote_path, is_active): 8 | 9 | is_js = (content_type in JS_MIMETYPES or remote_path.lower().endswith('.js')) 10 | 11 | if is_js: 12 | 13 | params = urlencode({ 14 | 'js_code': filedata, 15 | 'compilation_level': 'SIMPLE_OPTIMIZATIONS', 16 | 'output_info': 'compiled_code', 17 | 'output_format': 'text', 18 | }) 19 | 20 | conn = httplib.HTTPConnection('closure-compiler.appspot.com') 21 | conn.request('POST', '/compile', params, HEADERS) 22 | response = conn.getresponse() 23 | data = response.read() 24 | conn.close 25 | 26 | return data -------------------------------------------------------------------------------- /mediasync/processors/slim.py: -------------------------------------------------------------------------------- 1 | try: 2 | import slimmer 3 | SLIMMER_INSTALLED = True 4 | except ImportError: 5 | SLIMMER_INSTALLED = False 6 | 7 | def css_minifier(filedata, content_type, remote_path, is_active): 8 | is_css = content_type == 'text/css' or remote_path.lower().endswith('.css') 9 | if SLIMMER_INSTALLED and is_active and is_css: 10 | return slimmer.css_slimmer(filedata) 11 | 12 | def js_minifier(filedata, content_type, remote_path, is_active): 13 | is_js = content_type == 'text/javascript' or remote_path.lower().endswith('.js') 14 | if SLIMMER_INSTALLED and is_active and is_js: 15 | return slimmer.css_slimmer(filedata) 16 | -------------------------------------------------------------------------------- /mediasync/processors/yuicompressor.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from mediasync import CSS_MIMETYPES, JS_MIMETYPES 3 | import os 4 | from subprocess import Popen, PIPE 5 | 6 | def _yui_path(settings): 7 | if not hasattr(settings, 'MEDIASYNC'): 8 | return None 9 | path = settings.MEDIASYNC.get('YUI_COMPRESSOR_PATH', None) 10 | if path: 11 | path = os.path.realpath(os.path.expanduser(path)) 12 | return path 13 | 14 | def css_minifier(filedata, content_type, remote_path, is_active): 15 | is_css = (content_type in CSS_MIMETYPES or remote_path.lower().endswith('.css')) 16 | yui_path = _yui_path(settings) 17 | if is_css and yui_path and is_active: 18 | proc = Popen(['java', '-jar', yui_path, '--type', 'css'], stdout=PIPE, 19 | stderr=PIPE, stdin=PIPE) 20 | stdout, stderr = proc.communicate(input=filedata) 21 | return str(stdout) 22 | 23 | def js_minifier(filedata, content_type, remote_path, is_active): 24 | is_js = (content_type in JS_MIMETYPES or remote_path.lower().endswith('.js')) 25 | yui_path = _yui_path(settings) 26 | if is_js and yui_path and is_active: 27 | proc = Popen(['java', '-jar', yui_path, '--type', 'js'], stdout=PIPE, 28 | stderr=PIPE, stdin=PIPE) 29 | stdout, stderr = proc.communicate(input=filedata) 30 | return str(stdout) 31 | -------------------------------------------------------------------------------- /mediasync/signals.py: -------------------------------------------------------------------------------- 1 | from django.core import management 2 | from django.core.management.base import CommandError 3 | from django.dispatch import Signal 4 | from mediasync import SyncException, listdir_recursive 5 | from mediasync.conf import msettings 6 | import os 7 | import subprocess 8 | 9 | pre_sync = Signal() 10 | post_sync = Signal() 11 | 12 | def collectstatic_receiver(sender, **kwargs): 13 | try: 14 | management.call_command('collectstatic') 15 | except CommandError: 16 | raise SyncException("collectstatic management command not found") 17 | 18 | def sass_receiver(sender, **kwargs): 19 | 20 | sass_cmd = msettings.get("SASS_COMMAND", "sass") 21 | 22 | root = msettings['STATIC_ROOT'] 23 | 24 | for filename in listdir_recursive(root): 25 | 26 | if filename.endswith('.sass') or filename.endswith('.scss'): 27 | 28 | sass_path = os.path.join(root, filename) 29 | css_path = sass_path[:-4] + "css" 30 | 31 | cmd = "%s %s %s" % (sass_cmd, sass_path, css_path) 32 | subprocess.call(cmd.split(' ')) -------------------------------------------------------------------------------- /mediasync/templatetags/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunlightlabs/django-mediasync/aa8ce4cfff757bbdb488463c64c0863cca6a1932/mediasync/templatetags/__init__.py -------------------------------------------------------------------------------- /mediasync/templatetags/media.py: -------------------------------------------------------------------------------- 1 | from django import template 2 | from mediasync import backends 3 | from mediasync.conf import msettings 4 | import mediasync 5 | import mimetypes 6 | 7 | # Instance of the backend you configured in settings.py. 8 | client = backends.client() 9 | 10 | register = template.Library() 11 | 12 | class BaseTagNode(template.Node): 13 | """ 14 | Base class for all mediasync nodes. 15 | """ 16 | def __init__(self, path): 17 | super(BaseTagNode, self).__init__() 18 | # This is the filename or path+filename supplied by the template call. 19 | self.path = path 20 | 21 | def is_secure(self, context): 22 | """ 23 | Looks at the RequestContext object and determines if this page is 24 | secured with SSL. Linking unencrypted media on an encrypted page will 25 | show a warning icon on some browsers. We need to be able to serve from 26 | an encrypted source for encrypted pages, if our backend supports it. 27 | 'django.core.context_processors.request' must be added to 28 | TEMPLATE_CONTEXT_PROCESSORS in settings.py. 29 | """ 30 | return 'request' in context and context['request'].is_secure() 31 | 32 | def supports_gzip(self, context): 33 | """ 34 | Looks at the RequestContext object and determines if the client 35 | supports gzip encoded content. If the client does, we will send them 36 | to the gzipped version of files that are allowed to be compressed. 37 | Clients without gzip support will be served the original media. 38 | """ 39 | if 'request' in context and client.supports_gzip(): 40 | enc = context['request'].META.get('HTTP_ACCEPT_ENCODING', '') 41 | return 'gzip' in enc and msettings['SERVE_REMOTE'] 42 | return False 43 | 44 | def get_media_url(self, context): 45 | """ 46 | Checks to see whether to use the normal or the secure media source, 47 | depending on whether the current page view is being sent over SSL. 48 | The USE_SSL setting can be used to force HTTPS (True) or HTTP (False). 49 | 50 | NOTE: Not all backends implement SSL media. In this case, they'll just 51 | return an unencrypted URL. 52 | """ 53 | use_ssl = msettings['USE_SSL'] 54 | is_secure = use_ssl if use_ssl is not None else self.is_secure(context) 55 | return client.media_url(with_ssl=True) if is_secure else client.media_url() 56 | 57 | def mkpath(self, url, path, filename=None, gzip=False): 58 | """ 59 | Assembles various components to form a complete resource URL. 60 | 61 | args: 62 | url: (str) A base media URL. 63 | path: (str) The path on the host (specified in 'url') leading up 64 | to the file. 65 | filename: (str) The file name to serve. 66 | gzip: (bool) True if client should receive *.gzt version of file. 67 | """ 68 | if path: 69 | url = "%s/%s" % (url.rstrip('/'), path.strip('/')) 70 | 71 | if filename: 72 | url = "%s/%s" % (url, filename.lstrip('/')) 73 | 74 | content_type = mimetypes.guess_type(url)[0] 75 | if gzip and content_type in mediasync.TYPES_TO_COMPRESS: 76 | url = "%s.gzt" % url 77 | 78 | cb = msettings['CACHE_BUSTER'] 79 | if cb: 80 | # Cache busters help tell the client to re-download the file after 81 | # a change. This can either be a callable or a constant defined 82 | # in settings.py. 83 | cb_val = cb(url) if callable(cb) else cb 84 | url = "%s?%s" % (url, cb_val) 85 | 86 | return msettings['URL_PROCESSOR'](url) 87 | 88 | def resolve_path(self, context): 89 | if self.path: 90 | try: 91 | path = template.Variable(self.path).resolve(context) 92 | except template.VariableDoesNotExist: 93 | path = self.path 94 | return path 95 | 96 | def get_path_from_tokens(token): 97 | """ 98 | Just yanks the path out of a list of template tokens. Ignores any 99 | additional arguments. 100 | """ 101 | tokens = token.split_contents() 102 | 103 | if len(tokens) > 1: 104 | # At least one argument. Only interested in the path, though. 105 | return tokens[1].strip("\"'") 106 | else: 107 | # No path provided in the tag call. 108 | return None 109 | 110 | def media_url_tag(parser, token): 111 | """ 112 | If msettings['SERVE_REMOTE'] == False, returns your STATIC_URL. 113 | When msettings['SERVE_REMOTE'] == True, returns your storage 114 | backend's remote URL (IE: S3 URL). 115 | 116 | If an argument is provided with the tag, it will be appended on the end 117 | of your media URL. 118 | 119 | *NOTE:* This tag returns a URL, not any kind of HTML tag. 120 | 121 | Usage:: 122 | 123 | {% media_url ["path/and/file.ext"] %} 124 | 125 | Examples:: 126 | 127 | {% media_url %} 128 | {% media_url "images/bunny.gif" %} 129 | {% media_url %}/themes/{{ theme_variable }}/style.css 130 | """ 131 | return MediaUrlTagNode(get_path_from_tokens(token)) 132 | register.tag('media_url', media_url_tag) 133 | 134 | class MediaUrlTagNode(BaseTagNode): 135 | """ 136 | Node for the {% media_url %} tag. See the media_url_tag method above for 137 | documentation and examples. 138 | """ 139 | def render(self, context): 140 | path = self.resolve_path(context) 141 | media_url = self.get_media_url(context) 142 | 143 | if not path: 144 | # No path provided, just return the base media URL. 145 | return media_url 146 | else: 147 | # File/path provided, return the assembled URL. 148 | return self.mkpath(media_url, path, gzip=self.supports_gzip(context)) 149 | 150 | """ 151 | # CSS related tags 152 | """ 153 | 154 | def css_tag(parser, token): 155 | """ 156 | Renders a tag to include the stylesheet. It takes an optional second 157 | parameter for the media attribute; the default media is "screen, projector". 158 | 159 | Usage:: 160 | 161 | {% css ".css" [""] %} 162 | 163 | Examples:: 164 | 165 | {% css "myfile.css" %} 166 | {% css "myfile.css" "screen, projection"%} 167 | """ 168 | path = get_path_from_tokens(token) 169 | 170 | tokens = token.split_contents() 171 | if len(tokens) > 2: 172 | # Get the media types from the tag call provided by the user. 173 | media_type = tokens[2][1:-1] 174 | else: 175 | # Default values. 176 | media_type = "screen, projection" 177 | 178 | return CssTagNode(path, media_type=media_type) 179 | register.tag('css', css_tag) 180 | 181 | def css_print_tag(parser, token): 182 | """ 183 | Shortcut to render CSS as a print stylesheet. 184 | 185 | Usage:: 186 | 187 | {% css_print "myfile.css" %} 188 | 189 | Which is equivalent to 190 | 191 | {% css "myfile.css" "print" %} 192 | """ 193 | path = get_path_from_tokens(token) 194 | # Hard wired media type, since this is for media type of 'print'. 195 | media_type = "print" 196 | 197 | return CssTagNode(path, media_type=media_type) 198 | register.tag('css_print', css_print_tag) 199 | 200 | class CssTagNode(BaseTagNode): 201 | """ 202 | Node for the {% css %} tag. See the css_tag method above for 203 | documentation and examples. 204 | """ 205 | def __init__(self, *args, **kwargs): 206 | super(CssTagNode, self).__init__(*args) 207 | self.media_type = kwargs.get('media_type', "screen, projection") 208 | 209 | def render(self, context): 210 | path = self.resolve_path(context) 211 | media_url = self.get_media_url(context) 212 | css_path = msettings['CSS_PATH'] 213 | joined = msettings['JOINED'] 214 | 215 | if msettings['SERVE_REMOTE'] and path in joined: 216 | # Serving from S3/Cloud Files. 217 | return self.linktag(media_url, css_path, path, self.media_type, context) 218 | elif not msettings['SERVE_REMOTE'] and msettings['EMULATE_COMBO']: 219 | # Don't split the combo file into its component files. Emulate 220 | # the combo behavior, but generate/serve it locally. Useful for 221 | # testing combo CSS before deploying. 222 | return self.linktag(media_url, css_path, path, self.media_type, context) 223 | else: 224 | # If this is a combo file seen in the JOINED key on the 225 | # MEDIASYNC dict, break it apart into its component files and 226 | # write separate tags for each. 227 | filenames = joined.get(path, (path,)) 228 | return ' '.join((self.linktag(media_url, css_path, fn, self.media_type, context) for fn in filenames)) 229 | 230 | def linktag(self, url, path, filename, media, context): 231 | """ 232 | Renders a tag for the stylesheet(s). 233 | """ 234 | if msettings['DOCTYPE'] == 'xhtml': 235 | markup = """""" 236 | elif msettings['DOCTYPE'] == 'html5': 237 | markup = """""" 238 | else: 239 | markup = """""" 240 | return markup % (self.mkpath(url, path, filename, gzip=self.supports_gzip(context)), media) 241 | 242 | """ 243 | # JavaScript related tags 244 | """ 245 | 246 | def js_tag(parser, token): 247 | """ 248 | Renders a tag to include a JavaScript file. 249 | 250 | Usage:: 251 | 252 | {% js "somefile.js" %} 253 | 254 | """ 255 | return JsTagNode(get_path_from_tokens(token)) 256 | register.tag('js', js_tag) 257 | 258 | class JsTagNode(BaseTagNode): 259 | """ 260 | Node for the {% js %} tag. See the js_tag method above for 261 | documentation and examples. 262 | """ 263 | def render(self, context): 264 | path = self.resolve_path(context) 265 | media_url = self.get_media_url(context) 266 | js_path = msettings['JS_PATH'] 267 | joined = msettings['JOINED'] 268 | 269 | if msettings['SERVE_REMOTE'] and path in joined: 270 | # Serving from S3/Cloud Files. 271 | return self.scripttag(media_url, js_path, path, context) 272 | elif not msettings['SERVE_REMOTE'] and msettings['EMULATE_COMBO']: 273 | # Don't split the combo file into its component files. Emulate 274 | # the combo behavior, but generate/serve it locally. Useful for 275 | # testing combo JS before deploying. 276 | return self.scripttag(media_url, js_path, path, context) 277 | else: 278 | # If this is a combo file seen in the JOINED key on the 279 | # MEDIASYNC dict, break it apart into its component files and 280 | # write separate tags for each. 281 | filenames = joined.get(path, (path,)) 282 | return ' '.join((self.scripttag(media_url, js_path, fn, context) for fn in filenames)) 283 | 284 | def scripttag(self, url, path, filename, context): 285 | """ 286 | Renders a """ 290 | else: 291 | markup = """""" 292 | return markup % self.mkpath(url, path, filename, gzip=self.supports_gzip(context)) 293 | -------------------------------------------------------------------------------- /mediasync/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunlightlabs/django-mediasync/aa8ce4cfff757bbdb488463c64c0863cca6a1932/mediasync/tests/__init__.py -------------------------------------------------------------------------------- /mediasync/tests/media/_test/joined.css: -------------------------------------------------------------------------------- 1 | body { margin: 1px; } 2 | p { margin: 5px; } 3 | -------------------------------------------------------------------------------- /mediasync/tests/media/_test/joined.js: -------------------------------------------------------------------------------- 1 | function one() { 2 | alert(1); 3 | }; 4 | function two() { 5 | alert(2); 6 | }; 7 | -------------------------------------------------------------------------------- /mediasync/tests/media/css/1.css: -------------------------------------------------------------------------------- 1 | body { margin: 1px; } -------------------------------------------------------------------------------- /mediasync/tests/media/css/2.css: -------------------------------------------------------------------------------- 1 | p { margin: 5px; } -------------------------------------------------------------------------------- /mediasync/tests/media/css/3.scss: -------------------------------------------------------------------------------- 1 | a { 2 | color: #ce4dd6; 3 | &:hover { color: #ffb3ff; } 4 | &:visited { color: #c458cb; } 5 | } -------------------------------------------------------------------------------- /mediasync/tests/media/img/black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/sunlightlabs/django-mediasync/aa8ce4cfff757bbdb488463c64c0863cca6a1932/mediasync/tests/media/img/black.png -------------------------------------------------------------------------------- /mediasync/tests/media/js/1.js: -------------------------------------------------------------------------------- 1 | function one() { 2 | alert(1); 3 | } -------------------------------------------------------------------------------- /mediasync/tests/media/js/2.js: -------------------------------------------------------------------------------- 1 | function two() { 2 | alert(2); 3 | } -------------------------------------------------------------------------------- /mediasync/tests/models.py: -------------------------------------------------------------------------------- 1 | # no models, but needed for tests.py -------------------------------------------------------------------------------- /mediasync/tests/run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export AWS_KEY=$1 4 | export AWS_SECRET=$2 5 | 6 | django-admin.py test --settings=mediasync.tests.settings --pythonpath=../.. -------------------------------------------------------------------------------- /mediasync/tests/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | TEST_ROOT = os.path.abspath(os.path.dirname(__file__)) 3 | 4 | DATABASE_ENGINE = 'sqlite3' 5 | DATABASE_NAME = 'mediasynctest.db' 6 | 7 | STATIC_ROOT = os.path.join(TEST_ROOT, 'media') 8 | STATIC_URL = '/media/' 9 | 10 | MEDIASYNC = { 11 | 'BACKEND': 'mediasync.backends.dummy', 12 | } 13 | 14 | INSTALLED_APPS = ('mediasync','mediasync.tests',) 15 | -------------------------------------------------------------------------------- /mediasync/tests/tests.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.core.exceptions import ImproperlyConfigured 3 | from django.dispatch import receiver 4 | from django.template import Context, Template 5 | from hashlib import md5 6 | import glob 7 | import httplib 8 | import itertools 9 | import os 10 | import re 11 | import time 12 | import unittest 13 | 14 | from mediasync import backends, JS_MIMETYPES, listdir_recursive 15 | from mediasync.backends import BaseClient 16 | from mediasync.conf import msettings 17 | from mediasync.signals import pre_sync, post_sync, sass_receiver 18 | import mediasync 19 | import mimetypes 20 | 21 | PWD = os.path.abspath(os.path.dirname(__file__)) 22 | 23 | EXPIRES_RE = re.compile(r'^\w{3}, \d{2} \w{3} \d{4} \d{2}:\d{2}:\d{2} GMT$') 24 | 25 | def readfile(path): 26 | f = open(path, 'r') 27 | content = f.read() 28 | f.close() 29 | return content 30 | 31 | class Client(BaseClient): 32 | 33 | def __init__(self, *args, **kwargs): 34 | super(Client, self).__init__(*args, **kwargs) 35 | 36 | def put(self, filedata, content_type, remote_path, force=False): 37 | if hasattr(self, 'put_callback'): 38 | return self.put_callback(filedata, content_type, remote_path, force) 39 | else: 40 | return True 41 | 42 | def remote_media_url(self, with_ssl=False): 43 | return ('https' if with_ssl else 'http') + "://localhost" 44 | 45 | # 46 | # tests 47 | # 48 | 49 | class BackendTestCase(unittest.TestCase): 50 | 51 | def setUp(self): 52 | msettings['BACKEND'] = 'not.a.backend' 53 | 54 | def tearDown(self): 55 | msettings['BACKEND'] = 'mediasync.backends.dummy' 56 | 57 | def testInvalidBackend(self): 58 | self.assertRaises(ImproperlyConfigured, backends.client) 59 | 60 | class MockClientTestCase(unittest.TestCase): 61 | 62 | def setUp(self): 63 | msettings['BACKEND'] = 'mediasync.tests.tests' 64 | msettings['PROCESSORS'] = [] 65 | msettings['SERVE_REMOTE'] = True 66 | msettings['JOINED'] = { 67 | 'css/joined.css': ('css/1.css', 'css/2.css'), 68 | 'js/joined.js': ('js/1.js', 'js/2.js'), 69 | } 70 | self.client = backends.client() 71 | 72 | def tearDown(self): 73 | msettings['JOINED'] = {} 74 | 75 | def testLocalMediaURL(self): 76 | self.assertEqual(self.client.get_local_media_url(), "/media/") 77 | 78 | def testMediaRoot(self): 79 | root = getattr(settings, 'STATIC_ROOT', None) 80 | if root is None: 81 | root = getattr(settings, 'MEDIA_ROOT', None) 82 | self.assertEqual(self.client.get_media_root(), root) 83 | 84 | def testMediaURL(self): 85 | self.assertEqual(self.client.media_url(with_ssl=False), "http://localhost") 86 | self.assertEqual(self.client.media_url(with_ssl=True), "https://localhost") 87 | 88 | def testSyncableDir(self): 89 | # not syncable 90 | self.assertFalse(mediasync.is_syncable_dir(".test")) 91 | self.assertFalse(mediasync.is_syncable_dir("_test")) 92 | # syncable 93 | self.assertTrue(mediasync.is_syncable_dir("test")) 94 | self.assertTrue(mediasync.is_syncable_dir("1234")) 95 | 96 | def testSyncableFile(self): 97 | # not syncable 98 | self.assertFalse(mediasync.is_syncable_file(".test")) 99 | self.assertFalse(mediasync.is_syncable_file("_test")) 100 | # syncable 101 | self.assertTrue(mediasync.is_syncable_file("test")) 102 | self.assertTrue(mediasync.is_syncable_file("1234")) 103 | 104 | def testDirectoryListing(self): 105 | allowed_files = [ 106 | 'css/1.css', 107 | 'css/2.css', 108 | 'css/3.scss', 109 | 'img/black.png', 110 | 'js/1.js', 111 | 'js/2.js', 112 | ] 113 | media_dir = os.path.join(PWD, 'media') 114 | listed_files = list(mediasync.listdir_recursive(media_dir)) 115 | self.assertListEqual(allowed_files, listed_files) 116 | 117 | def testSync(self): 118 | 119 | to_sync = { 120 | 'css/1.css': 'text/css', 121 | 'css/2.css': 'text/css', 122 | 'css/3.scss': msettings['DEFAULT_MIMETYPE'], 123 | 'css/joined.css': 'text/css', 124 | 'img/black.png': 'image/png', 125 | 'js/1.js': 'application/javascript', 126 | 'js/2.js': 'application/javascript', 127 | 'js/joined.js': 'application/javascript', 128 | } 129 | 130 | def generate_callback(is_forced): 131 | def myput(filedata, content_type, remote_path, force=is_forced): 132 | 133 | self.assertEqual(content_type, to_sync[remote_path]) 134 | self.assertEqual(force, is_forced) 135 | 136 | if remote_path in msettings['JOINED']: 137 | original = readfile(os.path.join(PWD, 'media', '_test', remote_path.split('/')[1])) 138 | else: 139 | args = [PWD, 'media'] + remote_path.split('/') 140 | original = readfile(os.path.join(*args)) 141 | 142 | self.assertEqual(filedata, original) 143 | 144 | return myput 145 | 146 | # normal sync 147 | self.client.put_callback = generate_callback(is_forced=False) 148 | mediasync.sync(self.client, force=False, verbose=False) 149 | 150 | # forced sync 151 | self.client.put_callback = generate_callback(is_forced=True) 152 | mediasync.sync(self.client, force=True, verbose=False) 153 | 154 | class S3ClientTestCase(unittest.TestCase): 155 | 156 | def setUp(self): 157 | 158 | bucket_hash = md5("%i-%s" % (int(time.time()), os.environ['USER'])).hexdigest() 159 | self.bucket_name = 'mediasync_test_' + bucket_hash 160 | 161 | msettings['BACKEND'] = 'mediasync.backends.s3' 162 | msettings['AWS_BUCKET'] = self.bucket_name 163 | msettings['AWS_KEY'] = os.environ['AWS_KEY'] or None 164 | msettings['AWS_SECRET'] = os.environ['AWS_SECRET'] or None 165 | msettings['PROCESSORS'] = [] 166 | msettings['SERVE_REMOTE'] = True 167 | msettings['JOINED'] = { 168 | 'css/joined.css': ('css/1.css', 'css/2.css'), 169 | 'js/joined.js': ('js/1.js', 'js/2.js'), 170 | } 171 | 172 | self.client = backends.client() 173 | 174 | def testServeRemote(self): 175 | 176 | msettings['SERVE_REMOTE'] = False 177 | self.assertEqual(backends.client().media_url(), '/media') 178 | 179 | msettings['SERVE_REMOTE'] = True 180 | self.assertEqual(backends.client().media_url(), 'http://s3.amazonaws.com/%s' % self.bucket_name) 181 | 182 | def testSync(self): 183 | 184 | # calculate cache control 185 | cc = "max-age=%i, public" % (self.client.expiration_days * 24 * 3600) 186 | 187 | # do a sync then reopen client 188 | mediasync.sync(self.client, force=True, verbose=False) 189 | self.client.open() 190 | conn = self.client.get_connection() 191 | 192 | # setup http connection 193 | http_conn = httplib.HTTPSConnection('s3.amazonaws.com') 194 | 195 | # test synced files then delete them 196 | bucket = conn.get_bucket(self.bucket_name) 197 | 198 | static_paths = mediasync.listdir_recursive(os.path.join(PWD, 'media')) 199 | joined_paths = msettings['JOINED'].iterkeys() 200 | 201 | for path in itertools.chain(static_paths, joined_paths): 202 | 203 | key = bucket.get_key(path) 204 | 205 | if path in msettings['JOINED']: 206 | args = [PWD, 'media', '_test', path.split('/')[1]] 207 | else: 208 | args = [PWD, 'media'] + path.split('/') 209 | local_content = readfile(os.path.join(*args)) 210 | 211 | # compare file content 212 | self.assertEqual(key.read(), local_content) 213 | 214 | # verify checksum 215 | key_meta = key.get_metadata('mediasync-checksum') or '' 216 | s3_checksum = key_meta.replace(' ', '+') 217 | (hexdigest, b64digest) = mediasync.checksum(local_content) 218 | self.assertEqual(s3_checksum, b64digest) 219 | 220 | # do a HEAD request on the file 221 | http_conn.request('HEAD', "/%s/%s" % (self.bucket_name, path)) 222 | response = http_conn.getresponse() 223 | response.read() 224 | 225 | # verify valid content type 226 | content_type = mimetypes.guess_type(path)[0] or msettings['DEFAULT_MIMETYPE'] 227 | self.assertEqual(response.getheader("Content-Type", None), content_type) 228 | 229 | # check for valid expires headers 230 | expires = response.getheader("Expires", None) 231 | self.assertRegexpMatches(expires, EXPIRES_RE) 232 | 233 | # check for valid cache control header 234 | cc_header = response.getheader("Cache-Control", None) 235 | self.assertEqual(cc_header, cc) 236 | 237 | # done with the file, delete it from S3 238 | key.delete() 239 | 240 | if content_type in mediasync.TYPES_TO_COMPRESS: 241 | 242 | key = bucket.get_key("%s.gzt" % path) 243 | 244 | # do a HEAD request on the file 245 | http_conn.request('HEAD', "/%s/%s.gzt" % (self.bucket_name, path)) 246 | response = http_conn.getresponse() 247 | response.read() 248 | 249 | key_meta = key.get_metadata('mediasync-checksum') or '' 250 | s3_checksum = key_meta.replace(' ', '+') 251 | self.assertEqual(s3_checksum, b64digest) 252 | 253 | key.delete() 254 | 255 | http_conn.close() 256 | 257 | # wait a moment then delete temporary bucket 258 | time.sleep(2) 259 | conn.delete_bucket(self.bucket_name) 260 | 261 | # close client 262 | self.client.close() 263 | 264 | def testMissingBucket(self): 265 | del msettings['AWS_BUCKET'] 266 | self.assertRaises(AssertionError, backends.client) 267 | 268 | class ProcessorTestCase(unittest.TestCase): 269 | 270 | def setUp(self): 271 | msettings['SERVE_REMOTE'] = True 272 | msettings['BACKEND'] = 'mediasync.tests.tests' 273 | msettings['PROCESSORS'] = ( 274 | 'mediasync.processors.slim.css_minifier', 275 | 'mediasync.processors.slim.js_minifier', 276 | lambda fd, ct, rp, r: fd.upper(), 277 | ) 278 | self.client = backends.client() 279 | 280 | def testJSProcessor(self): 281 | 282 | try: 283 | import slimmer 284 | except ImportError: 285 | self.skipTest("slimmer not installed, skipping test") 286 | 287 | content = """var foo = function() { 288 | alert(1); 289 | };""" 290 | 291 | ct = 'text/javascript' 292 | procd = self.client.process(content, ct, 'test.js') 293 | self.assertEqual(procd, 'VAR FOO = FUNCTION(){ALERT(1)};') 294 | 295 | def testCSSProcessor(self): 296 | 297 | try: 298 | import slimmer 299 | except ImportError: 300 | self.skipTest("slimmer not installed, skipping test") 301 | 302 | content = """html { 303 | border: 1px solid #000000; 304 | font-family: "Helvetica", "Arial", sans-serif; 305 | }""" 306 | 307 | ct = 'text/css' 308 | procd = self.client.process(content, ct, 'test.css') 309 | self.assertEqual(procd, 'HTML{BORDER:1PX SOLID #000;FONT-FAMILY:"HELVETICA","ARIAL",SANS-SERIF}') 310 | 311 | def testCustomProcessor(self): 312 | procd = self.client.process('asdf', 'text/plain', 'asdf.txt') 313 | self.assertEqual(procd, "ASDF") 314 | 315 | class ClosureCompilerTestCase(unittest.TestCase): 316 | 317 | def setUp(self): 318 | msettings['SERVE_REMOTE'] = True 319 | msettings['BACKEND'] = 'mediasync.tests.tests' 320 | msettings['PROCESSORS'] = ( 321 | 'mediasync.processors.closurecompiler.compile', 322 | ) 323 | self.client = backends.client() 324 | 325 | def testCompiler(self): 326 | 327 | content = """var foo = function() { 328 | alert(1); 329 | };""" 330 | 331 | for ct in JS_MIMETYPES: 332 | procd = self.client.process(content, ct, 'test.js') 333 | self.assertEqual(procd, 'var foo=function(){alert(1)};\n') 334 | 335 | def testNotJavascript(self): 336 | 337 | content = """html { 338 | border: 1px solid #000000; 339 | font-family: "Helvetica", "Arial", sans-serif; 340 | }""" 341 | 342 | procd = self.client.process(content, 'text/css', 'test.css') 343 | self.assertEqual(procd, content) 344 | 345 | class SignalTestCase(unittest.TestCase): 346 | 347 | def setUp(self): 348 | msettings['BACKEND'] = 'mediasync.tests.tests' 349 | self.client = backends.client() 350 | 351 | def tearDown(self): 352 | root = msettings['STATIC_ROOT'] 353 | for filename in glob.glob(os.path.join(root, "*/*.s[ac]ss")): 354 | path = filename[:-4] + "css" 355 | if os.path.exists(path): 356 | os.unlink(path) 357 | 358 | def testSyncSignals(self): 359 | 360 | self.client.called_presync = False 361 | self.client.called_postsync = False 362 | 363 | @receiver(pre_sync, weak=False) 364 | def presync_receiver(sender, **kwargs): 365 | self.assertEqual(self.client, sender) 366 | sender.called_presync = True 367 | 368 | @receiver(post_sync, weak=False) 369 | def postsync_receiver(sender, **kwargs): 370 | self.assertEqual(self.client, sender) 371 | sender.called_postsync = True 372 | 373 | mediasync.sync(self.client, force=True, verbose=False) 374 | 375 | self.assertTrue(self.client.called_presync) 376 | self.assertTrue(self.client.called_postsync) 377 | 378 | def testSassReceiver(self): 379 | 380 | pre_sync.connect(sass_receiver) 381 | 382 | mediasync.sync(self.client, force=True, verbose=False) 383 | 384 | root = msettings['STATIC_ROOT'] 385 | 386 | for sass_path in glob.glob(os.path.join(root, "*/*.s[ac]ss")): 387 | css_path = sass_path[:-4] + "css" 388 | self.assertTrue(os.path.exists(css_path)) 389 | 390 | class TemplateTagTestCase(unittest.TestCase): 391 | 392 | def setUp(self): 393 | msettings['BACKEND'] = 'mediasync.tests.tests' 394 | msettings['DOCTYPE'] = 'html5' 395 | self.client = backends.client() 396 | 397 | def testMediaURLTag(self): 398 | 399 | pathvar = 'images/logo.png' 400 | c = Context({'pathvar': pathvar}) 401 | 402 | # base media url 403 | t = Template('{% load media %}{% media_url %}') 404 | self.assertEqual(t.render(c), "http://localhost") 405 | 406 | # media url with string argument 407 | t = Template('{%% load media %%}{%% media_url "%s" %%}' % pathvar) 408 | self.assertEqual(t.render(c), "http://localhost/images/logo.png") 409 | 410 | # media url with variable argument 411 | t = Template('{% load media %}{% media_url pathvar %}') 412 | self.assertEqual(t.render(c), "http://localhost/images/logo.png") 413 | 414 | def testCSSTag(self): 415 | 416 | pathvar = 'styles/reset.css' 417 | c = Context({'pathvar': pathvar}) 418 | 419 | # css tag with string argument 420 | t = Template('{%% load media %%}{%% css "%s" %%}' % pathvar) 421 | self.assertEqual( 422 | t.render(c), 423 | '' % pathvar) 424 | 425 | # css tag with string argument and explicit media type 426 | t = Template('{%% load media %%}{%% css "%s" "tv" %%}' % pathvar) 427 | self.assertEqual( 428 | t.render(c), 429 | '' % pathvar) 430 | 431 | # css tag with variable argument 432 | t = Template('{% load media %}{% css pathvar %}') 433 | self.assertEqual( 434 | t.render(c), 435 | '' % pathvar) 436 | 437 | # css tag with variable argument and explicit media type 438 | t = Template('{% load media %}{% css pathvar "tv" %}') 439 | self.assertEqual( 440 | t.render(c), 441 | '' % pathvar) 442 | 443 | def testJSTag(self): 444 | 445 | pathvar = 'scripts/jquery.js' 446 | c = Context({'pathvar': pathvar}) 447 | 448 | # js tag with string argument 449 | t = Template('{%% load media %%}{%% js "%s" %%}' % pathvar) 450 | self.assertEqual( 451 | t.render(c), 452 | '' % pathvar) 453 | 454 | # js tag with variable argument 455 | t = Template('{% load media %}{% js pathvar %}') 456 | self.assertEqual( 457 | t.render(c), 458 | '' % pathvar) 459 | 460 | def testMultipleTags(self): 461 | 462 | paths = ('scripts/1.js','scripts/2.js') 463 | c = Context({'paths': paths}) 464 | 465 | t = Template('{% load media %}{% for path in paths %}{% media_url path %}{% endfor %}') 466 | self.assertEqual( 467 | t.render(c), 468 | 'http://localhost/scripts/1.jshttp://localhost/scripts/2.js') 469 | -------------------------------------------------------------------------------- /mediasync/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | Mediasync can serve media locally when MEDIASYNC['SERVE_REMOTE'] == False. 3 | The following urlpatterns are shimmed in, in that case. 4 | """ 5 | from django.conf.urls import * 6 | from mediasync import backends 7 | 8 | client = backends.client() 9 | local_media_url = client.local_media_url.strip('/') 10 | 11 | urlpatterns = patterns('mediasync.views', 12 | url(r'^%s/(?P.*)$' % local_media_url, 'static_serve', 13 | {'client': client}), 14 | ) 15 | -------------------------------------------------------------------------------- /mediasync/views.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module contains views used to serve static media if 3 | msettings['SERVE_REMOTE'] == False. See mediasync.urls to see how 4 | these are shimmed in. 5 | 6 | The static_serve() function is where the party starts. 7 | """ 8 | from django.http import HttpResponse 9 | from django.shortcuts import redirect 10 | from django.views.static import serve 11 | from mediasync import combine_files 12 | from mediasync.conf import msettings 13 | 14 | def combo_serve(request, path, client): 15 | """ 16 | Handles generating a 'combo' file for the given path. This is similar to 17 | what happens when we upload to S3. Processors are applied, and we get 18 | the value that we would if we were serving from S3. This is a good way 19 | to make sure combo files work as intended before rolling out 20 | to production. 21 | """ 22 | joinfile = path 23 | sourcefiles = msettings['JOINED'][path] 24 | # Generate the combo file as a string. 25 | combo_data, dirname = combine_files(joinfile, sourcefiles, client) 26 | 27 | if path.endswith('.css'): 28 | mime_type = 'text/css' 29 | elif joinfile.endswith('.js'): 30 | mime_type = 'application/javascript' 31 | 32 | return HttpResponse(combo_data, mimetype=mime_type) 33 | 34 | def _form_key_str(path): 35 | """ 36 | Given a URL path, massage it into a key we can perform a lookup on the 37 | MEDIASYNC['JOINED'] dict with. 38 | 39 | This mostly involves figuring into account the CSS_PATH and JS_PATH 40 | settings, if they have been set. 41 | """ 42 | if path.endswith('.css'): 43 | media_path_prefix = msettings['CSS_PATH'] 44 | elif path.endswith('.js'): 45 | media_path_prefix = msettings['JS_PATH'] 46 | else: 47 | # This isn't a CSS/JS file, no combo for you. 48 | return None 49 | 50 | if media_path_prefix: 51 | # CS/JSS path prefix has been set. Factor that into the key lookup. 52 | if not media_path_prefix.endswith('/'): 53 | # We need to add this slash so we can lop it off the 'path' 54 | # variable, to match the value in the JOINED dict. 55 | media_path_prefix += '/' 56 | 57 | if path.startswith(media_path_prefix): 58 | # Given path starts with the CSS/JS media prefix. Lop this off 59 | # so we can perform a lookup in the JOINED dict. 60 | return path[len(media_path_prefix):] 61 | else: 62 | # Path is in a root dir, send along as-is. 63 | return path 64 | 65 | # No CSS/JS path prefix set. Keep it raw. 66 | return path 67 | 68 | def _find_combo_match(path): 69 | """ 70 | Calculate the key to check the MEDIASYNC['JOINED'] dict for, perform the 71 | lookup, and return the matching key string if a match is found. If no 72 | match is found, return None instead. 73 | """ 74 | key_str = _form_key_str(path) 75 | if not key_str: 76 | # _form_key_str() says this isn't even a CSS/JS file. 77 | return None 78 | 79 | if not msettings['JOINED'].has_key(key_str): 80 | # No combo file match found. Must be an single file. 81 | return None 82 | else: 83 | # Combo match found, return the JOINED key. 84 | return key_str 85 | 86 | def static_serve(request, path, client): 87 | """ 88 | Given a request for a media asset, this view does the necessary wrangling 89 | to get the correct thing delivered to the user. This can also emulate the 90 | combo behavior seen when SERVE_REMOTE == False and EMULATE_COMBO == True. 91 | """ 92 | 93 | if msettings['SERVE_REMOTE']: 94 | # We're serving from S3, redirect there. 95 | url = client.remote_media_url().strip('/') + '/%(path)s' 96 | return redirect(url, permanent=True) 97 | 98 | if not msettings['SERVE_REMOTE'] and msettings['EMULATE_COMBO']: 99 | # Combo emulation is on and we're serving media locally. Try to see if 100 | # the given path matches a combo file defined in the JOINED dict in 101 | # the MEDIASYNC settings dict. 102 | combo_match = _find_combo_match(path) 103 | if combo_match: 104 | # We found a combo file match. Combine it and serve the result. 105 | return combo_serve(request, combo_match, client) 106 | 107 | # No combo file, but we're serving locally. Use the standard (inefficient) 108 | # Django static serve view. 109 | 110 | resp = serve(request, path, document_root=client.media_root, show_indexes=True) 111 | try: 112 | resp.content = client.process(resp.content, resp['Content-Type'], path) 113 | except KeyError: 114 | # HTTPNotModifiedResponse lacks the "Content-Type" key. 115 | pass 116 | return resp 117 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto>=1.8d,<2.0b1 2 | slimmer==0.1.30 3 | python-cloudfiles==1.7.5 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from mediasync import __version__ 2 | from setuptools import setup, find_packages 3 | import os 4 | 5 | f = open(os.path.join(os.path.dirname(__file__), 'README.rst')) 6 | readme = f.read() 7 | f.close() 8 | 9 | setup( 10 | name='django-mediasync', 11 | version=__version__, 12 | description='Django static media development and distribution tools', 13 | long_description=readme, 14 | author='Jeremy Carbaugh', 15 | author_email='jcarbaugh@sunlightfoundation.com', 16 | url='http://github.com/sunlightlabs/django-mediasync/', 17 | packages=find_packages(), 18 | package_data={ 19 | 'mediasync': [ 20 | 'tests/media/*/*', 21 | ] 22 | }, 23 | install_requires=[ 24 | 'boto', 25 | ], 26 | license='BSD License', 27 | platforms=["any"], 28 | classifiers=[ 29 | 'Development Status :: 4 - Beta', 30 | 'Intended Audience :: Developers', 31 | 'License :: OSI Approved :: BSD License', 32 | 'Natural Language :: English', 33 | 'Operating System :: OS Independent', 34 | 'Programming Language :: Python', 35 | 'Environment :: Web Environment', 36 | ], 37 | ) 38 | --------------------------------------------------------------------------------