├── .gitignore
├── asset_bender
    ├── templatetags
    │   ├── __init__.py
    │   ├── test
    │   │   ├── __init__.py
    │   │   └── test_asset_bender_tags.py
    │   └── asset_bender_tags.py
    ├── __init__.py
    ├── templates
    │   └── asset_bender
    │   │   └── scaffold
    │   │       ├── end_of_body.html
    │   │       ├── head.html
    │   │       └── bender_boilerplate_js.html
    ├── http.py
    ├── test
    │   └── test_fetch_ab_url_with_retries.py
    └── bundling.py
├── MANIFEST.in
├── setup.py
├── LICENSE.txt
└── Readme.md


/.gitignore:
--------------------------------------------------------------------------------
1 | dist/
2 | 


--------------------------------------------------------------------------------
/asset_bender/templatetags/__init__.py:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/asset_bender/templatetags/test/__init__.py:
--------------------------------------------------------------------------------
1 | 
2 | 
3 | 


--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include *.txt
2 | include *.md
3 | global-include *.html


--------------------------------------------------------------------------------
/asset_bender/__init__.py:
--------------------------------------------------------------------------------
1 | 
2 | class AssetBenderException(Exception):
3 |     pass
4 | 


--------------------------------------------------------------------------------
/asset_bender/templates/asset_bender/scaffold/end_of_body.html:
--------------------------------------------------------------------------------
1 | {% if bender_scaffold %}
2 |   <!-- ASSET BENDER END OF BODY JS IS HERE -->
3 |   {{ bender_scaffold.footer_js_html|safe }}
4 | {% endif %}


--------------------------------------------------------------------------------
/asset_bender/templates/asset_bender/scaffold/head.html:
--------------------------------------------------------------------------------
 1 | {% if bender_scaffold %}
 2 |   <!-- ASSET BENDER HEADER CSS -->
 3 |   {{ bender_scaffold.header_css_html|safe }}
 4 | 
 5 |   {% if bender_scaffold.has_excess_stylesheets_for_IE %}
 6 |       {{ bender_scaffold.header_forced_import_css_html_for_IE|safe }} 
 7 |   {% endif %}
 8 | 
 9 |   <!-- ASSET BENDER HEADER JS -->
10 |   {{ bender_scaffold.header_js_html|safe }}
11 | {% endif %}
12 | 


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | from setuptools import setup, find_packages
 3 | 
 4 | setup(
 5 |     name='asset_bender',
 6 |     version='0.1.21',
 7 |     description="A django runtime implementation for Asset Bender",
 8 |     long_description=open('Readme.md').read(),
 9 |     author='HubSpot Dev Team',
10 |     author_email='devteam+asset_bender_django@hubspot.com',
11 |     url='https://github.com/HubSpot/asset_bender_django',
12 |     # download_url='https://github.com/HubSpot/',
13 |     license='LICENSE.txt',
14 |     packages=find_packages(),
15 |     include_package_data=True,
16 |     install_requires=[
17 |         'django>=1.3.0',
18 |         'hscacheutils>=0.1.8',
19 |         'requests>=1.1.0',
20 |     ],
21 | )
22 | 


--------------------------------------------------------------------------------
/asset_bender/templates/asset_bender/scaffold/bender_boilerplate_js.html:
--------------------------------------------------------------------------------
 1 | {% load json_filter %}
 2 | {% load asset_bender_tags %}
 3 | 
 4 | <script type="text/javascript" language="javascript">
 5 |   hns('hubspot.bender', {
 6 |     staticDomain: '{{ bender_domain }}' || '{{ static_v3_domain }}' || window.location.host,
 7 |     staticDomainPrefix: '{{ bender_domain_with_prefix }}' || '{{ static_v3_domain_with_prefix }}' || ('//' + window.location.host),
 8 | 
 9 |     currentProject: '{{ host_project_name }}',
10 |     currentProjectVersion: '{% bender_build_for host_project_name %}',
11 | 
12 |     depVersions: {{ bender_assets_instance.get_all_dependency_versions|json }},
13 |     depPathPrefixes: {{ bender_assets_instance.get_all_dependency_url_prefixes|json }}
14 |   });
15 | </script>
16 | 


--------------------------------------------------------------------------------
/asset_bender/templatetags/asset_bender_tags.py:
--------------------------------------------------------------------------------
 1 | from django import template
 2 | from asset_bender.bundling import get_static_url, get_static_build_version
 3 | 
 4 | register = template.Library()
 5 | 
 6 | @register.simple_tag(takes_context=True)
 7 | def bender_url(context, full_asset_path):
 8 |     return get_static_url(full_asset_path, template_context=context)
 9 | 
10 | @register.simple_tag(takes_context=True)
11 | def bender_build_for(context, project_name):
12 |     return get_static_build_version(project_name, template_context=context)
13 | 
14 | # Deprecated
15 | @register.simple_tag
16 | def static_url(static_path):
17 |     return get_static_url(static_path)
18 | 
19 | @register.simple_tag(takes_context=True)
20 | def static3_url(context, full_asset_path):
21 |     return get_static_url(full_asset_path, template_context=context)
22 | 


--------------------------------------------------------------------------------
/asset_bender/templatetags/test/test_asset_bender_tags.py:
--------------------------------------------------------------------------------
 1 | 
 2 | from nose.tools import eq_, ok_
 3 | 
 4 | from django.template import Context, Template
 5 | 
 6 | from hsdjango.testcase import HubSpotTestCase
 7 | from asset_bender import bundling
 8 | from asset_bender.bundling import _extract_project_name_from_path
 9 | 
10 | template_src = '''\
11 | {% load asset_bender_tags %}{% bender_url "my_project/static/my/path.html"  %}'''
12 | mock_url = 'http://staticdomain.com/my_project/static-1.3/my/path.html'
13 | 
14 | 
15 | def get_mock_static3_asset_url(asset_path):
16 |     eq_('my_project', _extract_project_name_from_path(asset_path))
17 |     return 'http://staticdomain.com/my_project/static-1.3/my/path.html'
18 | 
19 | class StaticTagsCase(HubSpotTestCase):
20 |     def test_static3_url(self):
21 |         self.lax_mock(bundling3a.Static3, 'get_static3_asset_url', get_mock_static3_asset_url)
22 |         t = Template(template_src)
23 |         c = Context({})
24 |         rendered = t.render(c)
25 |         eq_(mock_url, rendered)
26 | 


--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 | Copyright (c) 2013 HubSpot, Inc.
2 | 
3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4 | 
5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6 | 
7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
8 | 


--------------------------------------------------------------------------------
/Readme.md:
--------------------------------------------------------------------------------
 1 | ## Usage:
 2 | 
 3 | First, make sure you've installed asset_bender: https://github.com/HubSpot/asset_bender/tree/master
 4 | 
 5 | Secondly, make sure that `PROJ_NAME`, `PROJ_DIR`, `BENDER_S3_DOMAIN`, and `BENDER_CDN_DOMAIN` are set in your
 6 | settings. `PROJ_NAME` should match the name in static_conf.json and `PROJ_DIR` needs to point to the python
 7 | module path (via something like `PROJ_DIR = dirname(realpath(__file__))`). 
 8 | 
 9 | `BENDER_S3_DOMAIN` is the domain that points to your S3 bucket and `BENDER_CDN_DOMAIN` is the CDN domain in
10 | front of S3 (if you have one).
11 | 
12 | Thirdly, make sure that you've included these lines in your Manifest.in:
13 | 
14 |     global-include static_conf.json
15 |     global-include prebuilt_recursive_static_conf.json
16 | 
17 | 
18 | Next, in your app's context processor do:
19 | 
20 | ```python
21 | from django.template import RequestContext
22 | from asset_bender.bundling import BenderAssets
23 | 
24 | def my_context_processor(request):
25 |     context = RequestContext(request)
26 | 
27 |     bender_assets = BenderAssets([
28 |          'my_project/static/js/my_project_bundle.js', 
29 |          'my_project/static/css/my_project_bundle.css', 
30 | 
31 |          'some_library/static/js/some_library_bundle.js', 
32 |          'some_library/static/css/some_library_bundle.js'
33 | 
34 |          ... etc ...
35 |      ], request.GET)
36 | 
37 |      context.update(bender_assets.generate_context_dict())
38 |      return context
39 | ```
40 | 
41 | And lastly, in your base template you'll need to include these templates:
42 | 
43 | 
44 | ```html
45 | <head>
46 |     ...
47 |     {% include "asset_bender/scaffold/head.html" %}
48 |     ...
49 | </head>
50 | 
51 | <body>
52 |     ...
53 |     {% include "asset_bender/scaffold/end_of_body.html" %}
54 | </body>
55 | ```
56 | 
57 | 
58 | To manually include a particular static asset in your HTML, use the template tag:
59 |     
60 | ```
61 | {% load asset_bender_tags %}
62 | {% bender_url "project_name/static/js/my-file.js" %}
63 | ```
64 | 
65 | The tag will output a full url with the proper domain and version number (as specified by this projects's dependencies).


--------------------------------------------------------------------------------
/asset_bender/http.py:
--------------------------------------------------------------------------------
 1 | import logging
 2 | import requests
 3 | from requests import ConnectionError, HTTPError, Timeout
 4 | 
 5 | from asset_bender import AssetBenderException
 6 | 
 7 | logger = logging.getLogger(__name__)
 8 | 
 9 | 
10 | # For testing
11 | class FauxException(Exception):
12 |     pass
13 | 
14 | 
15 | def _download_url(url, timeout=10, **kwargs):
16 |     result = requests.get(url, timeout=timeout, **kwargs)
17 |     result.raise_for_status()
18 | 
19 |     return result
20 | 
21 | def fetch_ab_url_with_retries(url, retries=None, timeouts=None, **kwargs):
22 |     """
23 |     Calls download_url retries number of times unless a valid response is returned earlier. 
24 |     Each retry will have a timeout of timeouts[i - 1] where i is the attempt number.
25 | 
26 |     If you omit retries, it will be set to len(timeouts).
27 |     """
28 | 
29 |     attempt = 1
30 |     latest_result = None
31 | 
32 |     if retries is None and timeouts is None:
33 |         retries = 1
34 |         timeouts = [None]
35 |     elif retries is None:
36 |         retries = len(timeouts)
37 | 
38 |     while attempt <= retries:
39 |         timeout = timeouts[min(len(timeouts), attempt) - 1] 
40 | 
41 |         try:
42 |             latest_result = _download_url(url, timeout=timeout, **kwargs)
43 |             return latest_result
44 | 
45 |         except (ConnectionError, HTTPError, Timeout, FauxException) as e:
46 |             status_code = getattr(latest_result, 'status_code', None)
47 | 
48 |             # Warn an continue if there are retries
49 |             if attempt < retries:
50 |                 logger.warning(e)
51 | 
52 |             # Otherwise throw a wrapped error
53 |             elif status_code >= 500:
54 |                 logger.error(e)
55 |                 raise AssetBenderException("Server error from Asset Bender (%s) for: %s" % (status_code, url), e)
56 | 
57 |             elif status_code in (404, 410):
58 |                 logger.error(e)
59 |                 raise AssetBenderException("Url doesn't exist in Asset Bender (%s): %s" % (status_code, url), e)
60 | 
61 |             elif status_code is not None and (status_code >= 400 or status_code < 200):
62 |                 logger.error(e)
63 |                 raise AssetBenderException("Asset Bender returned error (%s) for: %s" % (status_code, url), e)
64 | 
65 |             else:
66 |                 logger.error(e)
67 |                 raise e
68 | 
69 |         attempt += 1
70 | 
71 |     return latest_result
72 | 
73 | 
74 | def _format_result_error(self, content):
75 |     if not content:
76 |         return '(No error body found)'
77 |     start_code_block = content.find('<pre>')
78 |     end_code_block = content.find('</pre>')
79 |     if start_code_block < 0 or end_code_block < start_code_block:
80 |         return content
81 |     error = content[start_code_block + 5:end_code_block]
82 |     # Unescape entities in the error
83 |     from BeautifulSoup import BeautifulStoneSoup, Tag
84 |     error = BeautifulStoneSoup(error, convertEntities=BeautifulStoneSoup.HTML_ENTITIES).contents[0]
85 |     if isinstance(error, Tag):
86 |         error = ''.join(error.contents)
87 |     return "Error from hs-static:\n\n%s" % error


--------------------------------------------------------------------------------
/asset_bender/test/test_fetch_ab_url_with_retries.py:
--------------------------------------------------------------------------------
  1 | from nose.tools import eq_, ok_
  2 | from nose.tools import assert_raises
  3 | 
  4 | from asset_bender import http
  5 | from asset_bender.http import fetch_ab_url_with_retries, FauxException
  6 | 
  7 | from requests import Response
  8 | 
  9 | def build_fetch_tester(times_to_fail=0, expected_timeouts=[1]):
 10 |     num_attempts = [0]
 11 | 
 12 |     def wrapper(url, timeout=None):
 13 |         attempt = num_attempts[0]
 14 | 
 15 |         eq_(timeout, expected_timeouts[attempt], msg="Unexpected timeout")
 16 | 
 17 |         if attempt < times_to_fail:
 18 |             num_attempts[0] += 1
 19 |             raise FauxException("Random exception at %i" % (attempt + 1))
 20 |         else:
 21 |             result = Response()
 22 |             result._content = "Success at %i" % (attempt + 1)
 23 |             result.status_code = 200
 24 | 
 25 |         num_attempts[0] += 1
 26 |         return result
 27 | 
 28 |     return wrapper
 29 | 
 30 | def setup():
 31 |     global download_url_orig
 32 |     download_url_orig = http._download_url
 33 | 
 34 | def teardown():
 35 |     http._download_url = download_url_orig
 36 | 
 37 | def test_no_failures():
 38 |     http._download_url = build_fetch_tester(times_to_fail=0, expected_timeouts=[1,2,5])
 39 |     result = fetch_ab_url_with_retries('faux_url', timeouts=[1])
 40 |     eq_(result.text, "Success at 1")
 41 | 
 42 |     http._download_url = build_fetch_tester(times_to_fail=0, expected_timeouts=[1,2,5])
 43 |     result = fetch_ab_url_with_retries('faux_url', timeouts=[1,2])
 44 |     eq_(result.text, "Success at 1")
 45 | 
 46 |     http._download_url = build_fetch_tester(times_to_fail=0, expected_timeouts=[1,2,5])
 47 |     result = fetch_ab_url_with_retries('faux_url', timeouts=[1,2,5])
 48 |     eq_(result.text, "Success at 1")
 49 | 
 50 | def test_one_failure():
 51 |     http._download_url = build_fetch_tester(times_to_fail=1, expected_timeouts=[1,2,3])
 52 | 
 53 |     try:
 54 |         result = fetch_ab_url_with_retries('faux_url', timeouts=[1])
 55 |         eq_("Error wasn't thrown", False)
 56 |     except Exception as e:
 57 |         eq_(e.message, "Random exception at 1")
 58 | 
 59 |     http._download_url = build_fetch_tester(times_to_fail=1, expected_timeouts=[1,2,3])
 60 |     result = fetch_ab_url_with_retries('faux_url', timeouts=[1, 2])
 61 |     eq_(result.text, "Success at 2")
 62 | 
 63 |     http._download_url = build_fetch_tester(times_to_fail=1, expected_timeouts=[1,2,3])
 64 |     result = fetch_ab_url_with_retries('faux_url', timeouts=[1, 2, 3])
 65 |     eq_(result.text, "Success at 2")
 66 | 
 67 | def test_two_failures():
 68 |     http._download_url = build_fetch_tester(times_to_fail=2, expected_timeouts=[1,2,3])
 69 | 
 70 |     try:
 71 |         result = fetch_ab_url_with_retries('faux_url', timeouts=[1])
 72 |         eq_("Error wasn't thrown", False)
 73 |     except Exception as e:
 74 |         eq_(e.message, "Random exception at 1")
 75 | 
 76 |     http._download_url = build_fetch_tester(times_to_fail=2, expected_timeouts=[1,2,3])
 77 | 
 78 |     try:
 79 |         result = fetch_ab_url_with_retries('faux_url', timeouts=[1, 2])
 80 |         eq_("Error wasn't thrown", False)
 81 |     except Exception as e:
 82 |         eq_(e.message, "Random exception at 2")
 83 | 
 84 |     http._download_url = build_fetch_tester(times_to_fail=2, expected_timeouts=[1,2,3])
 85 |     result = fetch_ab_url_with_retries('faux_url', timeouts=[1, 2, 3])
 86 |     eq_(result.text, "Success at 3")
 87 | 
 88 | def test_three_failures():
 89 |     http._download_url = build_fetch_tester(times_to_fail=3, expected_timeouts=[1,2,3])
 90 |     try:
 91 |         result = fetch_ab_url_with_retries('faux_url', timeouts=[1])
 92 |         eq_("Error wasn't thrown", False)
 93 |     except Exception as e:
 94 |         eq_(e.message, "Random exception at 1")
 95 | 
 96 |     http._download_url = build_fetch_tester(times_to_fail=3, expected_timeouts=[1,2,3])
 97 |     try:
 98 |         result = fetch_ab_url_with_retries('faux_url', timeouts=[1, 2])
 99 |         eq_("Error wasn't thrown", False)
100 |     except Exception as e:
101 |         eq_(e.message, "Random exception at 2")
102 | 
103 |     http._download_url = build_fetch_tester(times_to_fail=3, expected_timeouts=[1,2,3])
104 |     try:
105 |         result = fetch_ab_url_with_retries('faux_url', timeouts=[1, 2, 3])
106 |         eq_("Error wasn't thrown", False)
107 |     except Exception as e:
108 |         eq_(e.message, "Random exception at 3")
109 | 


--------------------------------------------------------------------------------
/asset_bender/bundling.py:
--------------------------------------------------------------------------------
  1 | 
  2 | import hashlib
  3 | import logging
  4 | import os
  5 | import re
  6 | import socket
  7 | import traceback
  8 | from itertools import izip_longest
  9 | 
 10 | try:
 11 |     import simplejson as json
 12 | except ImportError:
 13 |     import json
 14 | 
 15 | try:
 16 |     from hubspot.hsutils import get_setting, get_setting_default
 17 | except ImportError:
 18 |     from hscacheutils.setting_wrappers import get_setting, get_setting_default
 19 | 
 20 | from hscacheutils.raw_cache import MAX_MEMCACHE_TIMEOUT
 21 | from hscacheutils.generational_cache import CustomUseGenCache, DummyGenCache
 22 | 
 23 | from asset_bender.http import fetch_ab_url_with_retries
 24 | 
 25 | 
 26 | logger = logging.getLogger(__name__)
 27 | 
 28 | CSS_EXTENSIONS = ('css', 'sass', 'scss')
 29 | JS_EXTENSIONS  = ('js', 'coffee')
 30 | NON_PRECOMPILED_EXTENSIONS = ('js', 'css')
 31 | PRECOMPILED_EXTENSIONS = tuple([ext for ext in CSS_EXTENSIONS + JS_EXTENSIONS if ext not in NON_PRECOMPILED_EXTENSIONS])
 32 | 
 33 | TAG_FUNCTION_NAME = 'bender_asset_url_callback'
 34 | SCAFFOLD_CONTEXT_NAME = 'bender_scaffold'
 35 | BENDER_ASSETS_CONTEXT_NAME = 'bender_assets_instance'
 36 | STATIC_DOMAIN_CONTEXT_NAME = 'bender_domain'
 37 | STATIC_DOMAIN_WITH_PREFIX_CONTEXT_NAME = 'bender_domain_with_prefix'
 38 | HOST_PROJECT_CONTEXT_NAME = 'host_project_name'
 39 | 
 40 | FORCE_BUILD_PARAM_PREFIX = "forceBuildFor-"
 41 | HOST_NAME = socket.gethostname()
 42 | 
 43 | def get_bender_or_static3_setting(setting_name, default_value):
 44 |     static3_setting_name = setting_name.replace('BENDER_', 'STATIC3_')
 45 |     return get_setting_default(setting_name, get_setting_default(static3_setting_name, default_value))
 46 | 
 47 | 
 48 | LOG_CACHE_MISSES = get_bender_or_static3_setting('BENDER_LOG_CACHE_MISSES', True)
 49 | LOG_S3_FETCHES = get_bender_or_static3_setting('BENDER_LOG_S3_FETCHES', True)
 50 | 
 51 | 
 52 | def build_scaffold(request, included_bundles):
 53 |     return BenderAssets(included_bundles, request.GET).generate_scaffold()
 54 | 
 55 | def get_static_url(full_asset_path, template_context=None, bender_assets=None):
 56 |     '''
 57 |     Gets an absolute url with the correct build version for an asset of the given project name.
 58 | 
 59 |     Re-uses the BenderAssets instance on the template context so that we only have to hit memcached
 60 |     for the build versions once per request (or you can manually pass in an instance).
 61 |     '''
 62 |     bender_assets = _extract_bender_assets_instance_from_template_context(template_context, bender_assets)
 63 |     return bender_assets.get_bender_asset_url(full_asset_path)
 64 | 
 65 | def load_static_json_content(full_asset_path, template_context=None, bender_assets=None):
 66 |     bender_assets = _extract_bender_assets_instance_from_template_context(template_context, bender_assets)
 67 |     return bender_assets.fetch_bender_asset_contents(full_asset_path)
 68 | 
 69 | def get_static_build_version(project_name, template_context=None, bender_assets=None):
 70 |     bender_assets = _extract_bender_assets_instance_from_template_context(template_context, bender_assets)
 71 |     return bender_assets.get_static3_build_version(project_name)
 72 | 
 73 | def _extract_bender_assets_instance_from_template_context(template_context, bender_assets=None):
 74 |     if template_context == None and bender_assets == None:
 75 |         logger.warning("No template_context or bender_assets instance passed, that will probably cause lots of excess memcache requests")
 76 |     elif bender_assets == None:
 77 |         bender_assets = template_context.get(BENDER_ASSETS_CONTEXT_NAME)
 78 | 
 79 |     if not bender_assets:
 80 |         bender_assets = BenderAssets()
 81 | 
 82 |     if template_context and not template_context.get(BENDER_ASSETS_CONTEXT_NAME):
 83 |         template_context.set(BENDER_ASSETS_CONTEXT_NAME, bender_assets)
 84 | 
 85 |     return bender_assets
 86 | 
 87 | 
 88 | def _is_only_on_qa():
 89 |     return get_setting('ENV') == 'qa'
 90 | 
 91 | 
 92 | # This will force different cache keys per build, which is desirable,
 93 | # because new builds may have different versions set in static_conf.json
 94 | _key_base = os.environ.get('BUILD_NUM', '') or os.environ.get('HS_JENKINS_BUILD_NUM', '')
 95 | 
 96 | project_version_cache = CustomUseGenCache([
 97 |     'static_build_name_for:project',
 98 |     'static_deps_for_project:host_project',
 99 |     'static_deps_for_project_%s:host_project' % _key_base
100 |     ],
101 |     timeout=MAX_MEMCACHE_TIMEOUT)
102 | 
103 | scaffold_cache = CustomUseGenCache([
104 |     'bender_all_scaffolds',
105 |     'bender_scaffold_for_project:scaffold_key',
106 |     'static3_scaffold_for_project_%s:scaffold_key' % _key_base
107 |     ],
108 |     timeout=MAX_MEMCACHE_TIMEOUT)
109 | 
110 | if get_bender_or_static3_setting('BENDER_NO_CACHE', False):
111 |     project_version_cache = DummyGenCache()
112 |     scaffold_cache = DummyGenCache()
113 | 
114 | def invalidate_cache_for_deploy(project_name):
115 |     '''
116 |     Invalidates the Asset Bender versions for this project. Do this as a part of your build
117 |     and/or deploy scripts to force a live running app to resolve its versions anew (and run against
118 |     the latest front-end code).
119 |     '''
120 |     project_version_cache.invalidate('static_build_name_for:project', project=project_name)
121 |     project_version_cache.invalidate('static_deps_for_project:host_project', host_project=project_name)  # For backwards compatibility
122 |     project_version_cache.invalidate('static_deps_for_project_%s:host_project' % _key_base, host_project=project_name)
123 |     scaffold_cache.invalidate('bender_all_scaffolds')
124 | 
125 | 
126 | class BenderAssets(object):
127 |     def __init__(self, bundle_paths=(), http_get_params=None, exclude_default_bundles=False):
128 |         '''
129 |         @bundle_paths - a list containing the paths of the bundles to include
130 |         @http_get_params - the request.GET query dictionary
131 |         '''
132 |         http_get_params = http_get_params if http_get_params else {}
133 |         self.is_debug = self._check_is_debug_mode(http_get_params)
134 |         self.use_local_daemon = self._check_use_local_daemon(http_get_params)
135 |         self.skip_scaffold_cache = False
136 | 
137 |         self.included_bundle_paths = []
138 | 
139 |         # Used for the cases when you don't want to include the default bundles (style_guide)
140 |         if not exclude_default_bundles:
141 |             self.included_bundle_paths.extend(get_setting_default('DEFAULT_ASSET_BENDER_BUNDLES', get_setting_default('DEFAULT_BUNDLES_V3', [])))
142 | 
143 |         self.included_bundle_paths.extend(bundle_paths)
144 | 
145 |         # strip first slashes for consistency
146 |         self.included_bundle_paths = [path.lstrip('/') for path in self.included_bundle_paths]
147 | 
148 |         self.host_project_name = get_setting_default('PROJ_NAME', None)
149 | 
150 |         forced_build_version_by_project = self._extract_forced_versions_from_params(http_get_params)
151 |         self.s3_fetcher = S3BundleFetcher(self.host_project_name, self.is_debug, forced_build_version_by_project)
152 |         is_local_debug = self.is_debug
153 | 
154 |         if get_bender_or_static3_setting('BENDER_LOCAL_PROJECT_MODE', False):
155 |             is_local_debug = True
156 | 
157 |         self.local_daemon_fetcher = LocalDaemonBundleFetcher(self.host_project_name, is_local_debug, forced_build_version_by_project)
158 | 
159 |     def generate_context_dict(self):
160 |         '''
161 |         Helper to get the variables you need to exist in your request context for Asset Bender
162 |         '''
163 |         return {
164 |             SCAFFOLD_CONTEXT_NAME: self.generate_scaffold(),
165 |             STATIC_DOMAIN_CONTEXT_NAME: self.get_domain(),
166 |             BENDER_ASSETS_CONTEXT_NAME: self,
167 |             STATIC_DOMAIN_WITH_PREFIX_CONTEXT_NAME: self.get_prefixed_domain(),
168 |             HOST_PROJECT_CONTEXT_NAME: self.host_project_name,
169 |         }
170 | 
171 |     def generate_scaffold(self):
172 |         '''
173 |         The primary public method that will be called from the project's context_processor
174 | 
175 |         Either gets the Scaffold object from the cache or dispatches to actually building
176 |         the scaffold from the the included bundles
177 |         '''
178 |         cache_key = self._get_scaffold_cache_key()
179 |         scaffold = None
180 | 
181 |         # We don't cache the scaffold during local development or when ?forceBuildFor-<project> params are used
182 |         if not self.use_local_daemon and not self.skip_scaffold_cache:
183 |             scaffold = scaffold_cache.get(scaffold_key=cache_key)
184 | 
185 |             if not scaffold and LOG_CACHE_MISSES:
186 |                 logger.debug("Asset Bender scaffold cache miss: %s" % cache_key)
187 | 
188 |         if not scaffold:
189 |             scaffold = self._generate_scaffold_without_cache()
190 | 
191 |         if not self.use_local_daemon and not self.skip_scaffold_cache:
192 |             scaffold_cache.set(scaffold, scaffold_key=cache_key)
193 | 
194 |         return scaffold
195 | 
196 |     def _get_scaffold_cache_key(self):
197 |         '''
198 |         The key is a hash of all the data the scaffold needs to be uniqued by
199 |         '''
200 |         # we include the host name since when we deploy a project to one node, it might have a version
201 |         # of the static bundles that is ahead of nodes that have not recieved a deploy yet
202 |         # we include the __file__ name so that every deploy will clear the cache (since it will have a new virtuvalenv path)
203 |         args = self.included_bundle_paths + [self.host_project_name] + [str(self.is_debug)] + [str(self.use_local_daemon)] \
204 |                + [HOST_NAME] + [__file__]
205 | 
206 |         long_key = '-'.join(args)
207 |         key = hashlib.md5(long_key).hexdigest()
208 |         return key
209 | 
210 |     def _generate_scaffold_without_cache(self):
211 |         self._validate_configuration()
212 |         scaffold = Scaffold()
213 | 
214 |         for bundle_path in self.included_bundle_paths:
215 |             self._add_bundle_to_scaffold(bundle_path, scaffold)
216 | 
217 |         return scaffold
218 | 
219 |     def _add_bundle_to_scaffold(self, bundle_path, scaffold, wrapper_template=None):
220 |             html = ''
221 |             contains_hardcoded_version = '/static-' in bundle_path
222 | 
223 |             if not contains_hardcoded_version and (self.use_local_daemon or self._check_use_local_daemon_for_project(bundle_path)):
224 |                 html = self.local_daemon_fetcher.fetch_include_html(bundle_path)
225 | 
226 |                 if not html:
227 |                     logger.error("Couldn't find bundle in local daemon: %s" % bundle_path)
228 | 
229 |             # If not using daemon, or if the html was not found in the daemon, then we check S3
230 |             if not html:
231 |                 html = self.s3_fetcher.fetch_include_html(bundle_path)
232 | 
233 |             if wrapper_template:
234 |                 html = wrapper_template % html
235 | 
236 |             if html:
237 |                 scaffold.add_html_by_file_name(bundle_path, html)
238 |             else:
239 |                 logger.error("Unknown bundle couldn't be added to scaffold: %s" % bundle_path)
240 | 
241 |     def invalidate_scaffold_cache(self):
242 |         cache_key = self._get_scaffold_cache_key()
243 |         scaffold_cache.invalidate('bender_scaffold_for_project:scaffold_key', scaffold_key=cache_key)
244 | 
245 |     def get_bender_asset_url(self, full_asset_path):
246 |         '''
247 |         Builds a URL to the particulr CSS, JS, IMG, or other file that is
248 |         part of project_name.  The URL includes the proper domain and build information.
249 | 
250 |         '''
251 |         project_name = _extract_project_name_from_path(full_asset_path)
252 |         if not project_name:
253 |             raise Exception('Your path must be of the form: "<project_name>/static/js/whatever.js"')
254 | 
255 |         # Break the full path down to just the asset path (everything under static/)
256 |         asset_path = full_asset_path.replace("%s/static/" % project_name, '')
257 | 
258 |         # Make sure the path doesn't refer to a precompiled extension (since that won't actually exist on s3)
259 |         extension = _find_extension(full_asset_path, also_search_folder_name=False)
260 | 
261 |         if extension in PRECOMPILED_EXTENSIONS:
262 |             message = "You cannot use the '%s' extension in this static path: %s.\n You must use 'js' or 'css' (It will work locally, but it won't work on QA/prod)." % (extension, full_asset_path)
263 | 
264 |             if get_setting('ENV') == 'prod':
265 |                 logger.error(message)
266 |             else:
267 |                 raise Exception(message)
268 | 
269 |         # Dispatch to the correct fetcher
270 |         if self.use_local_daemon:
271 |             return self.local_daemon_fetcher.get_asset_url(project_name, asset_path)
272 |         else:
273 |             return self.s3_fetcher.get_asset_url(project_name, asset_path)
274 | 
275 |     def get_static3_build_version(self, project_name):
276 |         # Dispatch to the correct fetcher
277 |         if self.use_local_daemon:
278 |             return self.local_daemon_fetcher._fetch_build_version(project_name)
279 |         else:
280 |             return self.s3_fetcher._fetch_build_version(project_name)
281 | 
282 |     # This assumes that the build process has placed the precompiled file in the
283 |     # python egg for QA/prod
284 |     def fetch_bender_asset_contents(self, full_asset_path):
285 |         if self.use_local_daemon:
286 |             return self.local_daemon_fetcher.fetch_static_file_contents(full_asset_path)
287 |         else:
288 |             return self.s3_fetcher.fetch_static_file_contents(full_asset_path)
289 | 
290 |     def get_dependency_version_snapshot(self):
291 |         '''
292 |         Called by a special endpoint on QA by the deploy script.  Generates a snapshot
293 |         of the current version of each depency so that when we deploy to prod we are guaranteed
294 |         to use the same version we were using on QA
295 |         '''
296 |         return self.s3_fetcher.get_dependency_version_snapshot()
297 | 
298 |     def _validate_configuration(self):
299 |         if not self.host_project_name:
300 |             raise Exception("You must hav PROJ_NAME set to your project name in settings.py (eg: PROJ_NAME = \"example_app_whatever...\") !")
301 | 
302 |         for bundle_path in self.included_bundle_paths:
303 |             extension = _find_extension(bundle_path, also_search_folder_name=False)
304 | 
305 |             if not extension:
306 |                 raise Exception("You must include an extension in this static path: %s.\n The file must end in 'js' or 'css' (It will work locally, but it won't work on QA/prod)." % (bundle_path))
307 | 
308 |             elif extension in PRECOMPILED_EXTENSIONS:
309 |                 raise Exception("You cannot use the '%s' extension in a bundle path (%s), you must use 'js' or 'css' (It can work locally, but it won't work on QA/prod)." % (bundle_path, extension))
310 | 
311 |     def get_all_dependency_versions(self):
312 |         '''
313 |         Similar to `get_dependency_version_snapshot`, but doesn't only use the s3 fetcher
314 |         and automatically adds "-debug" if in ?hsDebug=true and prod/QA
315 |         '''
316 |         if self.use_local_daemon:
317 |             dep_versions = self.local_daemon_fetcher._fetch_all_dependency_versions()
318 |         else:
319 |             dep_versions = self.s3_fetcher._fetch_all_dependency_versions()
320 | 
321 |         if self.is_debug and not self.use_local_daemon:
322 |             for dep, version in dep_versions.items():
323 |                 dep_versions[dep] = version + "-debug"
324 | 
325 |         return dep_versions
326 | 
327 |     def get_all_dependency_url_prefixes(self):
328 |         '''
329 |         Similar to `get_dependency_version_snapshot`, but appends "/<project>/static-" to each
330 |         version so it is ready to be directly inserted into an URL
331 |         '''
332 |         dep_versions_with_static = self.get_all_dependency_versions()
333 |         dep_versions_with_prefix = {}
334 | 
335 |         for dep, version in dep_versions_with_static.items():
336 |             dep_versions_with_prefix[dep] = "/" + dep + "/" + version
337 | 
338 |         return dep_versions_with_prefix
339 | 
340 |     def _check_use_local_daemon_for_project(self, bundle_path):
341 |         if not get_bender_or_static3_setting('BENDER_LOCAL_PROJECT_MODE', False):
342 |             return False
343 |         if bundle_path.startswith(self.host_project_name + '/'):
344 |             return True
345 |         return False
346 | 
347 |     def _check_use_local_daemon(self, request):
348 |         use_local = get_bender_or_static3_setting('BENDER_LOCAL_MODE', None)
349 |         if use_local != None:
350 |             return use_local
351 |         if get_setting('ENV') in ('local',):
352 |             return True
353 |         return False
354 | 
355 |     def _check_is_debug_mode(self, http_get_params):
356 |         '''
357 |         Debug mode will include the expanded, unbundled, non-minifised assets
358 |         returns a Boolean
359 |         '''
360 |         hs_debug = http_get_params.get('hsDebug')
361 |         if hs_debug:
362 |             return hs_debug != 'false'
363 | 
364 |         local_mode = get_bender_or_static3_setting('BENDER_LOCAL_MODE', None)
365 |         if local_mode != None:
366 |             return local_mode
367 | 
368 |         debug_mode = get_bender_or_static3_setting('BENDER_DEBUG_MODE', None)
369 |         if debug_mode != None:
370 |             return debug_mode
371 | 
372 |         return  get_setting('ENV') in ('local',)
373 | 
374 |     def _extract_forced_versions_from_params(self, http_get_params):
375 |         """
376 |         Pulls out all url params that look like "&forceBuildFor-<project_name>=<build_name>" and
377 |         puts them into a dict such as:
378 | 
379 |         {
380 |             "<project_name>": <build_name>,
381 |             ...
382 |         }
383 | 
384 |         This enables us to dynamically force a specific static build for any project at runtime. So with this
385 |         you can test future major versions of static assets on QA concurrently with the current major version.
386 |         (Eg. test major version 2 while 1 is still the one running by default on QA/prod).
387 | 
388 |         For <build_name> you can specify generic values (like "current", "latest", or "3") as well as specific
389 |         build names (like "static-4.59").
390 |         """
391 |         forced_build_version_by_project = dict()
392 | 
393 |         for param, value in http_get_params.items():
394 |             if param.startswith(FORCE_BUILD_PARAM_PREFIX):
395 |                 project_name = param[len(FORCE_BUILD_PARAM_PREFIX):]
396 |                 forced_build_version_by_project[project_name] = value
397 | 
398 |                 # Always skip the scaffold cache if we are forcing a version
399 |                 self.skip_scaffold_cache = True
400 | 
401 |         if len(forced_build_version_by_project):
402 |             return forced_build_version_by_project
403 | 
404 |     def get_domain(self):
405 |         if self.use_local_daemon:
406 |             return self.local_daemon_fetcher.get_domain()
407 |         else:
408 |             return self.s3_fetcher.get_domain()
409 | 
410 |     def get_prefixed_domain(self):
411 |         if self.use_local_daemon:
412 |            return self.local_daemon_fetcher.get_prefixed_domain()
413 |         else:
414 |             return self.s3_fetcher.get_prefixed_domain()
415 | 
416 | class BundleFetcherBase(object):
417 |     '''
418 |     Base class from getting included assets from either the local Asset Bender server or S3
419 |     '''
420 |     src_or_href_regex = re.compile(r'((?:src|href)=([\'"]))([^\'"]+\2)')
421 | 
422 |     def __init__(self, host_project_name='', is_debug=False, forced_build_version_by_project=None):
423 |         '''
424 |         @host_project_name - the project name of the application that we are runing from
425 |         '''
426 |         self.host_project_name = host_project_name
427 |         self.is_debug = is_debug
428 |         self.project_directory = get_setting('PROJ_DIR')
429 | 
430 |         # We store the build versions locally in this object so we don't have to
431 |         # hit memcached dozens of times per request every time we call get_asset_url.
432 |         self.per_request_project_build_version_cache = {}
433 |         self._add_forced_versions_to_per_request_cache(forced_build_version_by_project)
434 | 
435 |     def fetch_include_html(self, bundle_path):
436 |         raise NotImplementedError("Implement me in a subclass")
437 | 
438 |     def get_asset_url(self, project_name, asset_path):
439 |         raise NotImplementedError("Implement me in a subclass")
440 | 
441 |     def get_domain(self):
442 |         raise NotImplementedError("Implement me in a subclass")
443 | 
444 |     def get_prefixed_domain(self):
445 |         domain = self.get_domain()
446 | 
447 |         if domain:
448 |             if domain.startswith('//') or domain.startswith('http:') or domain.startswith('https:'):
449 |                 return domain
450 |             else:
451 |                 return "//" + domain
452 |         else:
453 |             return domain
454 | 
455 |     def _append_static_domain_to_links(self, html):
456 |         # Use the CDN domain instead of directly pointing to the s3 domain.
457 |         # Only doing the switch at this point because the rest of the build fetching code
458 |         # needs to use the s3 domain (to not get borked by caching).
459 |         result = self.src_or_href_regex.sub(r'\1//%s\3' % self.get_domain(), html)
460 |         return result
461 | 
462 |     def _add_forced_versions_to_per_request_cache(self, forced_build_version_by_project):
463 |         """
464 |         Initialize the per request cache with any specific builds overrides on this
465 |         request (via FORCE_BUILD_PARAM_PREFIX)
466 |         """
467 |         if forced_build_version_by_project:
468 |             for dep_name, dep_value in forced_build_version_by_project.items():
469 |                 if self._is_specific_build_name(dep_value):
470 |                     self.per_request_project_build_version_cache[dep_name] = dep_value
471 |                 else:
472 |                     self.per_request_project_build_version_cache[dep_name] = self._fetch_version_from_version_pointer(dep_value, dep_name)
473 | 
474 |     def _is_specific_build_name(self, build_name):
475 |        return isinstance(build_name, basestring) and build_name.startswith('static-')
476 | 
477 |     def _fetch_all_dependency_versions(self):
478 |         project_name_to_version = {}
479 |         project_names = self._get_static_conf_data().get('deps', {}).keys() + [self.host_project_name]
480 |         for project_name in project_names:
481 |             version = self._fetch_build_version(project_name)
482 |             project_name_to_version[project_name] = version
483 |         return project_name_to_version
484 | 
485 |     def _get_version_from_static_conf(self, project_name):
486 |         deps = self._get_static_conf_data().get('deps', {})
487 | 
488 |         if not deps.get(project_name):
489 |             logger.error("Tried to find a dependency (%s) in static_conf.json, but it didn't exist. Your static_conf.json must include all the static dependencies that your project may reference." % project_name)
490 | 
491 |         return deps.get(project_name, 'current')
492 | 
493 |     def _get_static_conf_data(self):
494 |         path = os.path.join(self.project_directory, 'static/static_conf.json')
495 |         parents_path = os.path.join(self.project_directory, '../static/static_conf.json')
496 | 
497 |         if os.path.isfile(path):
498 |             return _load_json_file_with_cache(path, throw_exception_if=_is_only_on_qa)
499 |         elif os.path.isfile(parents_path):
500 |             return _load_json_file_with_cache(parents_path, throw_exception_if=_is_only_on_qa)
501 |         else:
502 |             return {}
503 | 
504 | class LocalDaemonBundleFetcher(BundleFetcherBase):
505 |     def fetch_include_html(self, bundle_path):
506 |         url = "http://%s/bundle%s/%s.html?from=%s" % \
507 |             (self.get_domain(),
508 |              '-expanded' if self.is_debug else '',
509 |              bundle_path,
510 |              self.host_project_name
511 |              )
512 | 
513 |         result = fetch_ab_url_with_retries(url, timeouts=[1, 5, 25])
514 |         html = self._append_static_domain_to_links(result.text)
515 | 
516 |         return html
517 | 
518 |     def get_asset_url(self, project_name, asset_path):
519 |         try:
520 |             build_version = self._fetch_build_version(project_name)
521 |             url = 'http://%s/%s/%s/%s' % (self.get_domain(),  project_name, build_version, asset_path)
522 |         except Exception:
523 |             traceback.print_exc()
524 |             # HACK until we have a fix for projects with '.' in the project name
525 |             url = 'http://%s/%s/static/%s' % (self.get_domain(),  project_name, asset_path)
526 |         return url
527 | 
528 |     def get_domain(self):
529 |         return get_bender_or_static3_setting('BENDER_DAEMON_DOMAIN', 'localhost:3333')
530 | 
531 |     def _fetch_build_version(self, project_name):
532 |         """
533 |         Fetching the build for a specific project from the daemon (if that hasn't already been cached this request)
534 |         """
535 |         build_version = self.per_request_project_build_version_cache.get(project_name)
536 | 
537 |         if build_version:
538 |             return build_version
539 | 
540 |         build_version = self._fetch_build_version_from_daemon(project_name)
541 |         self.per_request_project_build_version_cache[project_name] = build_version
542 | 
543 |         return build_version
544 | 
545 |     def _fetch_build_version_from_daemon(self, project_name):
546 |         url = "http://%s/builds/%s?from=%s" % \
547 |             (self.get_domain(),
548 |              project_name,
549 |              self.host_project_name)
550 | 
551 |         result = fetch_ab_url_with_retries(url, timeouts=[1, 2, 5])
552 |         return result.text
553 | 
554 |     def fetch_static_file_contents(self, static_path):
555 |         url = "http://%s/%s" % (self.get_domain(), static_path)
556 |         result = fetch_ab_url_with_retries(url, timeouts=[1, 2, 5])
557 |         return json.loads(result.text)
558 | 
559 | class S3BundleFetcher(BundleFetcherBase):
560 |     def fetch_include_html(self, bundle_path):
561 |         project_name, hardcoded_version, bundle_postfix_path = self._split_bundle_path(bundle_path)
562 | 
563 |         if hardcoded_version:
564 |             build_version = hardcoded_version
565 |         else:
566 |             build_version = self._fetch_build_version(project_name)
567 | 
568 |         url = 'http://%s/%s/%s/%s.bundle%s.html' % (
569 |             self.get_domain(),
570 |             project_name,
571 |             build_version,
572 |             bundle_postfix_path,
573 |             '-expanded' if self.is_debug else '')
574 | 
575 |         if LOG_S3_FETCHES:
576 |             logger.info("Fetching the bundle html (static versions) for %(bundle_path)s" % locals())
577 | 
578 |         result = fetch_ab_url_with_retries(url, timeouts=[1,2,5])
579 |         return self._append_static_domain_to_links(result.text)
580 | 
581 | 
582 |     def get_asset_url(self, project_name, asset_path):
583 |         build_version = self._fetch_build_version(project_name)
584 |         url = 'https://%s/%s/%s/%s' % (self.get_domain(),  project_name, build_version, asset_path)
585 |         return url
586 | 
587 |     def get_dependency_version_snapshot(self):
588 |         '''
589 |         For every project in the local static_conf.json,
590 |         get the current version of that dependency.
591 | 
592 |         returns a dictionary in the form project_name=>build version
593 |         '''
594 |         return self._fetch_all_dependency_versions()
595 | 
596 |     def make_url_to_pointer(self, pointer, project_name):
597 |         # if the version is just an integer, that represents a major version, and so we create
598 |         # the major version pointer
599 |         if str(pointer).isdigit():
600 |             pointer = 'latest-version-%s' % str(pointer)
601 | 
602 |         url = 'http://%s/%s/%s' % (self._get_non_cdn_domain(), project_name, pointer)
603 |         if get_setting('ENV') not in ('prod',):
604 |             url += '-qa'
605 |         return url
606 | 
607 |     project_name_re = re.compile(r'^/?([^/]+)/(static(?:-\d+.\d+)?)/(.*)')
608 | 
609 |     def _split_bundle_path(self, bundle_path):
610 |         match = self.project_name_re.match(bundle_path)
611 | 
612 |         if match.group(2).startswith('static-'):
613 |             hardcoded_version = match.group(2)
614 |         else:
615 |             hardcoded_version = None
616 | 
617 |         return match.group(1), hardcoded_version, match.group(3)
618 | 
619 |     def _fetch_build_version(self, project_name):
620 |         # Are there any fixed local versions?
621 |         build_version = self._fetch_local_project_build_version(project_name)
622 | 
623 |         if build_version:
624 |             return build_version
625 | 
626 |         # Next, try the per-request mini-cache
627 |         build_version = self.per_request_project_build_version_cache.get(project_name)
628 | 
629 |         if build_version:
630 |             return build_version
631 | 
632 |         # Try memcache
633 |         build_version = project_version_cache.get(
634 |             project=project_name,
635 |             host_project=self.host_project_name)
636 | 
637 |         if not build_version:
638 |             if LOG_CACHE_MISSES:
639 |                 logger.debug("Asset Bender build version cache miss: %s from %s" % (project_name, self.host_project_name))
640 | 
641 |             # Next try fetching directly from s3
642 |             build_version = self._fetch_build_version_without_cache(project_name)
643 | 
644 |         if not build_version:
645 |             raise BundleException("Could not find a build version for %s" % project_name)
646 | 
647 |         project_version_cache.set(
648 |             build_version,
649 |             project=project_name,
650 |             host_project=self.host_project_name)
651 | 
652 |         self.per_request_project_build_version_cache[project_name] = build_version
653 |         return build_version
654 | 
655 |     def _fetch_local_project_build_version(self, project_name):
656 |         '''
657 |         If this bundle_path is being included from the project we are currently running in,
658 |         then we always use the version in prebuilt_recursive_static_conf.json, if it exists,
659 |         that way the .html and .py code is always linked to the exact javascript version
660 |         and we don't get issues where javascript can be deployed before a node is updated
661 |         '''
662 |         build_version = ''
663 |         if project_name == self.host_project_name:
664 |             build_version = self._get_prebuilt_version(project_name)
665 |             if not build_version:
666 |                 build_version = os.environ.get('HS_BENDER_FORCED_BUILD_VERSION_%s' % project_name.upper())
667 |         return build_version
668 | 
669 |     def _fetch_build_version_without_cache(self, project_name):
670 |         static_conf_version = self._get_version_from_static_conf(project_name)
671 |         if self._is_specific_build_name(static_conf_version):
672 |             return static_conf_version
673 |         pointer_build_version = self._fetch_version_from_version_pointer(static_conf_version, project_name)
674 |         prebuilt_build_version = self._get_prebuilt_version(project_name)
675 |         frozen_by_deploy_version = self._get_frozen_at_deploy_version(project_name)
676 |         build_version = self._maximum_version_of(
677 |             pointer_build_version,
678 |             prebuilt_build_version,
679 |             frozen_by_deploy_version)
680 | 
681 |         if LOG_S3_FETCHES:
682 |             logger.info("Fetched static version for %(project_name)s: %(build_version)s (max of %(pointer_build_version)s, %(prebuilt_build_version)s, and %(frozen_by_deploy_version)s)" % locals())
683 | 
684 |         return build_version
685 | 
686 |     def _fetch_version_from_version_pointer(self, pointer, project_name):
687 |         '''
688 |         Pointer is either 'current' or 'edge'.  This method downloads the pointer
689 |         from S3 and gets the actual build version from it (ex. 1.4.123 )
690 |         '''
691 |         url = self.make_url_to_pointer(pointer, project_name)
692 |         result = fetch_ab_url_with_retries(url, timeouts=[1, 2, 5])
693 | 
694 |         if not result.text:
695 |             self._check_for_fetch_html_errors_and_raise_exception(result, url)
696 |             raise AssetBenderException("Invalid version file (empty) from: %s" % url)
697 | 
698 |         return result.text.strip()
699 | 
700 |     def _get_prebuilt_version(self, project_name):
701 |         '''
702 |         When a project is built in Jenkins to QA, we store the version of the bundle that existed
703 |         when it was built
704 |         '''
705 |         path = os.path.join(self.project_directory, 'static/prebuilt_recursive_static_conf.json')
706 |         data = _load_json_file_with_cache(path, throw_exception_if=_is_only_on_qa)
707 | 
708 |         # If this is the host project, get the build from the "build" key instead of the deps dict
709 |         if project_name == self.host_project_name:
710 |             if data.get('build'):
711 |                 return "static-%s" % data['build']
712 |             else:
713 |                 return ''
714 |         else:
715 |             return data.get('deps', {}).get(project_name, '')
716 | 
717 |     def _get_frozen_at_deploy_version(self, project_name):
718 |         '''
719 |         At the time we deploy to prod, we get a snapshot from QA of the exact version number
720 |         of all dependencies.  When we deploy to prod, we write out a file that includes
721 |         this version number of snapshot.  Prod will then always use 'at least' the version number
722 |         of the snapshot.  So there is never any danger of having working code on QA, then deploying
723 |         to prod only to find you are importing an old, buggy version of a dependency
724 |         '''
725 |         path = os.path.join(self.project_directory, 'static/frozen_at_deploy_version_snapshot.json')
726 |         return _load_json_file_with_cache(path).get(project_name, '')
727 | 
728 |     def _maximum_version_of(self, *args):
729 |         args = [a for a in args if a]
730 |         return sorted(args, cmp=self._compare_build_names).pop()
731 | 
732 |     def _compare_build_names(self, x_build, y_build):
733 |         """
734 |         Implementation of cmp() for build names. So:
735 | 
736 |         >>> compare_build_names('static-1.0', 'static-1.1')
737 |         -1
738 |         >>> compare_build_names('static-2.0', 'static-1.1')
739 |         1
740 |         >>> compare_build_names('static-3.4', 'static-3.4')
741 |         0
742 | 
743 |         """
744 |         # Convert each build name to a two element tuple
745 |         x, y = [tuple(map(int, build.replace('static-', '').split('.'))) for build in (x_build, y_build)]
746 |         major_cmp = cmp(x[0], y[0])
747 |         if major_cmp != 0:
748 |           return major_cmp
749 |         else:
750 |           return cmp(x[1], y[1])
751 | 
752 |     def get_domain(self):
753 |         return get_bender_or_static3_setting('BENDER_CDN_DOMAIN', 'static.hsappstatic.net')
754 | 
755 |     def _get_non_cdn_domain(self):
756 |         '''
757 |         When downloading the version pointer, we need to skip the CDN and go direct to avoid problems with caching
758 |         '''
759 |         return get_bender_or_static3_setting('BENDER_S3_DOMAIN', 'hubspot-static2cdn.s3.amazonaws.com')
760 | 
761 |     # Assumes that the build has placed the precomplied file in the python egg
762 |     # (oh and that it is a JSON file)
763 |     def fetch_static_file_contents(self, static_path):
764 |         filename = os.path.basename(static_path)
765 |         return _load_json_file_with_cache(filename)
766 | 
767 | 
768 | class Scaffold(object):
769 |     """
770 |     An object for holding a set of js and css paths
771 |     """
772 | 
773 |     # Lovely IE, http://john.albin.net/css/ie-stylesheets-not-loading ...
774 |     # Six minus the real max (31) so that one style tag can used
775 |     # to "@import" the rest, and we leave enough of a buffer for any
776 |     # js files that append link/style blocks later.
777 |     MAX_IE_CSS_INCLUDES = 20
778 | 
779 |     # Also, if that isn't fun enough, there is a max number of @imports in a
780 |     # single <style> element: http://blogs.msdn.com/b/ieinternals/archive/2011/05/14/internet-explorer-stylesheet-rule-selector-import-sheet-limit-maximum.aspx
781 |     MAX_IMPORTS_PER_STYLE_ELEMENT = 25
782 | 
783 |     head_template = "asset_bender/scaffold/head.html"
784 |     end_of_body_template = "asset_bender/scaffold/end_of_body.html"
785 | 
786 |     def __init__(self, force_normal_include=False):
787 |         '''
788 |         @head_js - a collection of javascript src files
789 |         @head_css - a collection of css files for the head
790 |         @footer_js - a collection of js files for the footer
791 |         '''
792 |         self.head_js = []
793 |         self.head_css = []
794 |         self.footer_js = []
795 | 
796 |         self.force_normal_include = force_normal_include
797 | 
798 |     def total_css_files(self):
799 |         return len(self.head_css)
800 | 
801 |     def add_head_css_html(self, html):
802 |         self.head_css += html.split('\n')
803 | 
804 |     def add_head_js_html(self, html):
805 |         self.head_js += html.split('\n')
806 | 
807 |     def add_footer_js_html(self, html):
808 |         self.footer_js += html.split('\n')
809 | 
810 |     def add_html_by_file_name(self, file_name, html):
811 |         '''
812 |         Adds the html to the proper section based on the name and extension of 'file_name'
813 |         '''
814 |         if _find_extension(file_name) in CSS_EXTENSIONS:
815 |             self.add_head_css_html(html)
816 |         else:
817 |             if '_head.js' in file_name or '-head.js' in file_name:
818 |                 self.add_head_js_html(html)
819 |             else:
820 |                 self.add_footer_js_html(html)
821 | 
822 | 
823 |     # Methods used by the layout templates to output scaffold files
824 |     def header_js_html(self):
825 |         html = "\n".join(self.head_js)
826 | 
827 |         # JS only for IE
828 |         html += """
829 |         """
830 | 
831 |         return html
832 | 
833 |     def footer_js_html(self):
834 |         return "\n".join(self.footer_js)
835 | 
836 |     def header_css_html(self):
837 |         if self.force_normal_include:
838 |             return "\n".join(self.head_css)
839 |         else:
840 |             return "\n".join(self.head_css[:self.MAX_IE_CSS_INCLUDES])
841 | 
842 |     def has_excess_stylesheets_for_IE(self):
843 |         return self.total_css_files() > self.MAX_IE_CSS_INCLUDES and not self.force_normal_include
844 | 
845 |     def header_forced_import_css_html_for_IE(self):
846 |         """
847 |         Get all of the excess css links that wouldn't work in IE and breaks them
848 |         into @imports in a separate <style> element.
849 | 
850 |         Also chunks things so there are less than 30 @imports per <style> element
851 |         (oh IE...)
852 |         """
853 | 
854 |         if self.has_excess_stylesheets_for_IE():
855 | 
856 |             # Convert all the excess <link> elements into @imports
857 |             import_lines = map(self._convert_link_to_import, self.head_css[self.MAX_IE_CSS_INCLUDES:])
858 | 
859 |             # Chunk those @imports by MAX_IMPORTS_PER_STYLE_ELEMENT, and then
860 |             # turn each chunk into a single string separated by newlines
861 |             chunked_import_lines = chunk(self.MAX_IMPORTS_PER_STYLE_ELEMENT, import_lines)
862 |             chunked_import_lines = ['\n'.join(filter(None, lines)) for lines in chunked_import_lines]
863 | 
864 |             # Join together into <style> tags
865 |             result = '\n\n'.join(["<style>\n%s\n</style>" % c for c in chunked_import_lines])
866 | 
867 |             return result
868 | 
869 |         else:
870 |             return ""
871 | 
872 |     def _convert_link_to_import(self, link_html):
873 |         """
874 |         Converts:
875 |             <link href="/style_guide/static/sass/style_guide_plus_layout.css?body=1" media="screen" rel="stylesheet" type="text/css" />
876 |         To:
877 |             @import "/style_guide/static/sass/style_guide_plus_layout.css?body=1";
878 |         """
879 | 
880 |         try:
881 |             # Get the URL via indexes
882 |             left_index = link_html.index('href=') + 6
883 |             quote_char = link_html[left_index - 1]
884 |             right_index = link_html.index(quote_char, left_index)
885 |             url = link_html[left_index:right_index]
886 |         except:
887 |             print "Warning, trying to add a non css file (link element) to the scaffold: %s" % link_html
888 |             return ""
889 | 
890 |         return "@import \"%s\";" % url
891 | 
892 | _file_json_cache = {}
893 | def _load_json_file_with_cache(path, throw_exception_if=None):
894 |     '''
895 |     Loads json data from the local file system and caches the result in local memory
896 |     '''
897 |     if path in _file_json_cache:
898 |         return _file_json_cache[path]
899 | 
900 |     if not os.path.isfile(path):
901 |         if hasattr(throw_exception_if, '__call__') and throw_exception_if() and not get_bender_or_static3_setting('BENDER_QA_EMULATION', False):
902 |             raise IOError("""
903 | Couldn't find the prebuilt static dependencies file at: %s
904 | You should double check that your static and jenkins config are correct. And that you have these lines in your Manifest.in:
905 | 
906 |         global-include static_conf.json
907 |         global-include prebuilt_recursive_static_conf.json
908 | 
909 | Note: this error only appears on QA (and is a warning so things don't unknowningly break on prod).
910 | 
911 | If you have any questions, you can bug tfinley@hubspot.com.
912 |                 """ % path)
913 |         else:
914 |             _file_json_cache[path] = {}
915 |             return {}
916 | 
917 |     with open(path, 'r') as f:
918 |         data = json.load(f)
919 |         f.close()
920 |         _file_json_cache[path] = data
921 |         return data
922 | 
923 | 
924 | path_extension_regex = re.compile(r'/(css|sass|scss|coffee|js)/')
925 | 
926 | def _find_extension(filename, also_search_folder_name=True):
927 |     extension = os.path.splitext(filename)[1]
928 |     # If there was an extension at the end of the file, grab it
929 |     # and strip off the leading period
930 |     if extension:
931 |         extension = extension[1:]
932 |     # Otherwise look for /<ext>/ in the path
933 |     elif also_search_folder_name:
934 |         match = path_extension_regex.search(filename)
935 |         if match:
936 |             extension = match.group(1)
937 |     if extension:
938 |         extension = extension.lower()
939 |     return extension
940 | 
941 | 
942 | project_name_static_path_regex = re.compile(r"(?:\/|^)([^\/]+)\/static\/")
943 | 
944 | def _extract_project_name_from_path(path_or_url):
945 |     """
946 |     Extracts the project_name out of a path or URL that looks like any of these:
947 |     project_name/static/...
948 |     .../project_name/static/...
949 |     """
950 |     match = project_name_static_path_regex.search(path_or_url)
951 |     if match:
952 |         return match.group(1)
953 |     else:
954 |         return None
955 | 
956 | 
957 | # Via http://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks-in-python
958 | def chunk(n, iterable, padvalue=None):
959 |     "chunk(3, 'abcdefg', 'x') --> ('a','b','c'), ('d','e','f'), ('g','x','x')"
960 |     return izip_longest(*[iter(iterable)]*n, fillvalue=padvalue)
961 | 


--------------------------------------------------------------------------------