Image caching
11 |
12 |
40 |
41 | 13 | TL;DR The interesting code is in scripts/bootstrap.js 14 |
15 | 16 |17 | This page loads 5 images. Three are specified in a remote manifest.json file, 18 | one is added from the img tag in the page and the final one comes from 19 | an object created in JavaScript. These are the three ways to get images. 20 |
21 | 22 |23 | Image 1 should never cache. That is, the headers that come back from the 24 | server should set no-cache for it. This triggers a warning, because we can't 25 | save the image for offline. 26 |
27 | 28 |29 | All other images are given somewhere between 60 and 180 seconds as their 30 | max-age. You can see the images saved in the filesystem along with an additional file 31 | stored in the root which contains the details on all files we've saved. 32 |
33 | 34 |35 | If the manifest or a file is removed, or a file has expired, the browser will request it 36 | again from the server. If it is not possible to retrieve a new copy of the 37 | file then the old one is served. 38 |
39 |
42 | ![]()
53 |
54 |
55 |
56 |
57 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
--------------------------------------------------------------------------------
/demo/placeholder.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/googlearchive/apps-resource-loader/5e15c184eabcc902bacf84f68d654988578ac5c2/demo/placeholder.png
--------------------------------------------------------------------------------
/demo/remote_images.json:
--------------------------------------------------------------------------------
1 | {
2 | "images": [{
3 | "src": "images/sample-1.png",
4 | "priority": 1
5 | },{
6 | "src": "images/sample-2.png",
7 | "priority": 2,
8 | "placeholder": "placeholder.png",
9 | "width": 376,
10 | "height": 201
11 | },{
12 | "src": "images/sample-3.png",
13 | "placeholder": "placeholder.png"
14 | }]
15 | }
16 |
--------------------------------------------------------------------------------
/demo/scripts/app.js:
--------------------------------------------------------------------------------
1 | var count = 1;
2 |
3 | function onCacheError(evt) {
4 |
5 | var fileInfo = evt.data;
6 | var log = document.getElementById('log');
7 |
8 | log.textContent += (count++) + ": " + evt.data.src + " is not cacheable:\n" +
9 | fileInfo.warnings.join("\n") + "\n\n";
10 |
11 | // hack to scroll the warning pane
12 | log.scrollTop = 1000000;
13 | }
14 |
15 | function onClick(evt) {
16 | this.load();
17 | }
18 |
19 | function showClear() {
20 | var clearButton = document.getElementById('clear');
21 | clearButton.addEventListener('click', function() {
22 | RAL.FileSystem.removeDir('slowimageserver.appspot.com', function() {
23 | console.log("All clear");
24 | });
25 | });
26 | }
27 |
28 | if(RAL.FileSystem.isReady()) {
29 | showClear();
30 | } else {
31 | RAL.FileSystem.registerOnReady(showClear);
32 | }
33 |
--------------------------------------------------------------------------------
/demo/scripts/bootstrap.js:
--------------------------------------------------------------------------------
1 | window.addEventListener('load', function() {
2 |
3 | /**
4 | * Fires off a request to get a remote manifest file
5 | */
6 | function init() {
7 |
8 | // get the local resources
9 | parseLocalImages();
10 |
11 | // start by loading the remote asset list
12 | RAL.Loader.load("remote_images.json",
13 | 'text', // xhr response type
14 | onSuccessfulRMGet,
15 | onFailedRMGet);
16 | }
17 |
18 | function onSuccessfulRMGet(data) {
19 | var toLoad = JSON.parse(data);
20 | var imageToLoad = null;
21 | var remoteImage = null;
22 | var container = document.getElementById('mid');
23 |
24 | for(var i = 0; i < toLoad.images.length; i++) {
25 |
26 | imageToLoad = toLoad.images[i];
27 |
28 | // next add in an image directly from the script
29 | // with the highest queue priority
30 | remoteImage = new RAL.RemoteImage(imageToLoad);
31 |
32 | // attach the new image element to the DOM
33 | // (we could've passed in an image to the RemoteImage call
34 | // but since we didn't it created an image element)
35 | container.appendChild(remoteImage.element);
36 |
37 | // listen for the image being loaded
38 | // and spit out a console message
39 | remoteImage.addEventListener('loaded',
40 | onRemoteImageLoaded.bind(remoteImage));
41 |
42 | // also listen for cache problems like 'no-cache'
43 | remoteImage.addEventListener('cacheerror',
44 | onCacheError);
45 |
46 | // add it to the queue
47 | RAL.Queue.add(remoteImage);
48 | }
49 |
50 | go();
51 | }
52 |
53 | function onRemoteImageLoaded() {
54 | console.log("I have loaded the image from remote: " + this.src);
55 | }
56 |
57 | function onFailedRMGet() {
58 | console.warn("Could not load remote manifest");
59 |
60 | // just go get what we have queued
61 | go();
62 | }
63 |
64 | /**
65 | * Gets the images that are embedded in the page itself
66 | * and creates an additional image asset directly
67 | */
68 | function parseLocalImages() {
69 |
70 | // grab all images in the page
71 | var images = document.querySelectorAll('img');
72 |
73 | for (var i = 0; i < images.length; i++) {
74 |
75 | // wrap the image in a remote image
76 | // object and pass it to the queue
77 | // - set the image priority to the reverse
78 | // of the querySelectorAll order
79 | var remoteImage = new RAL.RemoteImage({
80 | element: images[i],
81 | priority: i
82 | });
83 |
84 | // declared in app.js - lets us know if we
85 | // hit cache errors, e.g. no-cache
86 | remoteImage.addEventListener('cacheerror', onCacheError);
87 |
88 | RAL.Queue.add(remoteImage);
89 | }
90 |
91 | // next add in an image (sample-5.png) directly from the script
92 | // with the highest queue priority
93 | var finalImage = new RAL.RemoteImage({
94 | src: "images/sample-5.png",
95 | priority: RAL.Queue.getNextHighestPriority(),
96 | width: 376,
97 | height: 201
98 | });
99 |
100 | // just listen for the image being loaded
101 | // and add it into the document
102 | finalImage.addEventListener('loaded', function() {
103 | var img = finalImage.element;
104 | document.getElementById('mid').appendChild(img);
105 | });
106 |
107 | // add it to the queue
108 | RAL.Queue.add(finalImage);
109 | }
110 |
111 | function go() {
112 |
113 | // start the queue, maxed out
114 | // to 4 connections
115 | RAL.Queue.setMaxConnections(4);
116 | RAL.Queue.start();
117 | }
118 |
119 | // wait for the file manifest to be loaded
120 | // before attempting anything
121 | if(RAL.FileManifest.isReady()) {
122 | init();
123 | } else {
124 | RAL.FileManifest.registerOnReady(init);
125 | }
126 |
127 | });
128 |
--------------------------------------------------------------------------------
/demo/style.css:
--------------------------------------------------------------------------------
1 | html,
2 | body {
3 | font-family: "Helvetica Neue", Helvetica, Arial;
4 | background: #E2E2E2;
5 | color: #555;
6 | text-shadow: 0 1px 1px rgba(255,255,255,0.4);
7 | line-height: 140%;
8 | }
9 |
10 | h1 {
11 | color: #333;
12 | text-shadow: 0 1px 1px rgba(255,255,255,0.6);
13 | padding: 15px 0 0 10px;
14 | }
15 |
16 | h2 {
17 | color: #333;
18 | text-shadow: 0 1px 1px rgba(255,255,255,0.6);
19 | padding: 10px 0 5px 0;
20 | margin: 0;
21 | font-size: 16px;
22 | }
23 |
24 | #container {
25 | position: relative;
26 | }
27 |
28 | #left {
29 | position: absolute;
30 | top: 80px;
31 | left: 20px;
32 | width: 370px;
33 | font-size: 13px;
34 | line-height: 125%;
35 | }
36 |
37 | #right {
38 | position: absolute;
39 | top: 200px;
40 | right: 30px;
41 | width: 80px;
42 | }
43 |
44 | #left a, #right a {
45 | border-radius: 40px;
46 | width: 80px;
47 | height: 80px;
48 | text-align: center;
49 | display: block;
50 | background: #333;
51 | text-indent: -30000px;
52 | }
53 |
54 | #left a:after,
55 | #right a:after {
56 | content:'←';
57 | text-indent: 0;
58 | position: absolute;
59 | top: 0;
60 | left: 0;
61 | width: 80px;
62 | height: 80px;
63 | line-height: 70px;
64 | color: #FFF;
65 | text-shadow: 0 1px 1px rgba(0,0,0,0.3);
66 | font-size: 45px;
67 | }
68 |
69 | #right a:after {
70 | content:'→';
71 | }
72 |
73 | #mid {
74 | margin: 0 0 0 400px;
75 | }
76 |
77 | .resource {
78 | background: #FFF;
79 | padding: 5px;
80 | border-radius: 3px;
81 | box-shadow: 0 1px 2px rgba(0,0,0,0.4);
82 | }
83 |
84 | #image-holder {
85 | min-height: 415px;
86 | margin-bottom: 30px;
87 | }
88 |
89 | #warnings {
90 | width: 360px;
91 | height: 170px;
92 | position: fixed;
93 | bottom: 20px;
94 | left: 20px;
95 | }
96 |
97 | textarea {
98 | width: 90%;
99 | height: 120px;
100 | background: rgba(255,255,255,0.5);
101 | color: #333;
102 | border: 1px solid gray;
103 | }
104 |
105 | #clear {
106 | margin-bottom: 20px;
107 | }
108 |
--------------------------------------------------------------------------------
/lib/RAL/CacheParser.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Parses cache headers so that we can respect them.
3 | */
4 | RAL.CacheParser = (function() {
5 |
6 | /**
7 | * Parses headers to determine whether there is
8 | * anything in there that we need to know about, e.g.
9 | * no-cache or no-store.
10 | * @param {string} headers The XHR headers to parse.
11 | * @see http://www.mnot.net/cache_docs/
12 | * @returns {object} An object with the extracted expiry for the asset.
13 | */
14 | function parse(headers) {
15 |
16 | var rMaxAge = /max\-age=(\d+)/gi;
17 | var rNoCache = /Cache-Control:.*?no\-cache/gi;
18 | var rNoStore = /Cache-Control:.*?no\-store/gi;
19 | var rMustRevalidate = /Cache-Control:.*?must\-revalidate/gi;
20 | var rExpiry = /Expires:\s(.*)/gi;
21 |
22 | var warnings = [];
23 | var expires = rMaxAge.exec(headers);
24 | var useBy = Date.now();
25 |
26 | // check for no-store
27 | if(rNoStore.test(headers)) {
28 | warnings.push("Cache-Control: no-store is set");
29 | }
30 |
31 | // check for no-cache
32 | if(rNoCache.test(headers)) {
33 | warnings.push("Cache-Control: no-cache is set");
34 | }
35 |
36 | // check for must-revalidate
37 | if(rMustRevalidate.test(headers)) {
38 | warnings.push("Cache-Control: must-revalidate is set");
39 | }
40 |
41 | // if no max-age is set check
42 | // for an Expires value
43 | if(expires !== null) {
44 | useBy = Date.now() + (expires[1] * 1000);
45 | } else {
46 |
47 | // attempt to use the Expires: header
48 | expires = rExpiry.exec(headers);
49 |
50 | // if that fails warn
51 | if(expires !== null) {
52 | useBy = Date.parse(expires[1]);
53 | } else {
54 | warnings.push("Cache-Control: max-age and Expires: headers are not set");
55 | }
56 | }
57 |
58 | return {
59 | headers: headers,
60 | cacheable: (warnings.length === 0),
61 | useBy: useBy,
62 | warnings: warnings
63 | };
64 | }
65 |
66 | return {
67 | parse: parse
68 | };
69 |
70 | })();
71 |
--------------------------------------------------------------------------------
/lib/RAL/FileManifest.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Represents the internal log of all files that
3 | * have been cached for offline use.
4 | */
5 | RAL.FileManifest = (function() {
6 |
7 | var manifest = null,
8 | ready = false,
9 | readyListeners = [],
10 | saving = false,
11 | hasUpdated = false;
12 |
13 | /**
14 | * Determines if the manifest is ready for use.
15 | * @returns {boolean} If the manifest is ready.
16 | */
17 | function isReady() {
18 | return ready;
19 | }
20 |
21 | /**
22 | * Registers a listener for when the manifest is
23 | * ready for interaction.
24 | * @param {Function} listener The listener function.
25 | */
26 | function registerOnReady(listener) {
27 | readyListeners.push(listener);
28 | }
29 |
30 | /**
31 | * Gets the file information from the manifest.
32 | * @param {string} src The source URL of the asset.
33 | * @param {Function} callback The callback function to
34 | * call with the asset details.
35 | */
36 | function get(src, callback) {
37 | var cleanSrc = RAL.Sanitiser.cleanURL(src);
38 | var fileInfo = manifest[cleanSrc] || null;
39 | callback(fileInfo);
40 | }
41 |
42 | /**
43 | * Sets the file information in the manifest.
44 | * @param {string} src The source URL of the asset.
45 | * @param {object} info The information to store against the asset.
46 | * @param {Function} callback The callback function to
47 | * call once the asset details have been saved.
48 | */
49 | function set(src, info, callback) {
50 | var cleanSrc = RAL.Sanitiser.cleanURL(src);
51 | manifest[cleanSrc] = info;
52 | save(callback);
53 | }
54 |
55 | /**
56 | * Resets the manifest of files.
57 | */
58 | function reset() {
59 | manifest = {};
60 | save();
61 | }
62 |
63 | /**
64 | * Callback for when there is no manifest available.
65 | * @private
66 | */
67 | function onManifestUnavailable() {
68 | onManifestLoaded("{}");
69 | }
70 |
71 | /**
72 | * Callback for when there is no manifest has laded. Passes through the
73 | * registered ready callbacks and fires each in turn.
74 | * @param {string} The JSON representation of the manifest.
75 | * @private
76 | */
77 | function onManifestLoaded(fileData) {
78 |
79 | ready = true;
80 | manifest = JSON.parse(fileData);
81 |
82 | if (readyListeners.length) {
83 | var listener = readyListeners.length;
84 | while(listener--) {
85 | readyListeners[listener]();
86 | }
87 | }
88 | }
89 |
90 | /**
91 | * Saves the manifest file.
92 | * @private
93 | */
94 | function save(callback) {
95 |
96 | var blob = new Blob([
97 | JSON.stringify(manifest)
98 | ], {type: 'application/json'});
99 |
100 | // Called whether or not the file exists
101 | RAL.FileSystem.set("manifest.json", blob, function() {
102 | if(!!callback) {
103 | callback();
104 | }
105 | });
106 | }
107 |
108 | /**
109 | * Requests the manifest JSON file from the file system.
110 | */
111 | function init() {
112 | RAL.FileSystem.getDataAsText("manifest.json",
113 | onManifestLoaded,
114 | onManifestUnavailable);
115 | }
116 |
117 | // check if the file system is good to go. If not, then
118 | // flag that we want to know when it is.
119 | if (RAL.FileSystem.isReady()) {
120 | init();
121 | } else {
122 | RAL.FileSystem.registerOnReady(init);
123 | }
124 |
125 | return {
126 | isReady: isReady,
127 | registerOnReady: registerOnReady,
128 | get: get,
129 | set: set,
130 | reset: reset
131 | };
132 |
133 | })();
134 |
--------------------------------------------------------------------------------
/lib/RAL/FileSystem.js:
--------------------------------------------------------------------------------
1 | /**
2 | * FileSystem API wrapper. Makes extensive use of
3 | * the FileSystem code from Eric Bidelman.
4 | *
5 | * @see http://www.html5rocks.com/en/tutorials/file/filesystem/
6 | */
7 | RAL.FileSystem = (function() {
8 |
9 | var ready = false,
10 | readyListeners = [],
11 | root = null,
12 | callbacks = {
13 |
14 | /**
15 | * Generic error handler, simply warns the user
16 | */
17 | onError: function(e) {
18 | var msg = '';
19 |
20 | switch (e.code) {
21 | case FileError.QUOTA_EXCEEDED_ERR:
22 | msg = 'QUOTA_EXCEEDED_ERR';
23 | break;
24 | case FileError.NOT_FOUND_ERR:
25 | msg = 'NOT_FOUND_ERR';
26 | break;
27 | case FileError.SECURITY_ERR:
28 | msg = 'SECURITY_ERR';
29 | break;
30 | case FileError.INVALID_MODIFICATION_ERR:
31 | msg = 'INVALID_MODIFICATION_ERR';
32 | break;
33 | case FileError.INVALID_STATE_ERR:
34 | msg = 'INVALID_STATE_ERR';
35 | break;
36 | default:
37 | msg = 'Unknown Error';
38 | break;
39 | }
40 |
41 | console.error('Error: ' + msg, e);
42 | },
43 |
44 | /**
45 | * Callback for once the file system has been fired up.
46 | * Informs any listeners waiting on it.
47 | */
48 | onInitialised: function(fs) {
49 | root = fs.root;
50 | ready = true;
51 |
52 | if(readyListeners.length) {
53 | var listener = readyListeners.length;
54 | while(listener--) {
55 | readyListeners[listener]();
56 | }
57 | }
58 | }
59 | };
60 |
61 | /**
62 | * Determines if the file system is ready for use.
63 | * @returns {boolean} If the file system is ready.
64 | */
65 | function isReady() {
66 | return ready;
67 | }
68 |
69 | /**
70 | * Registers a listener for when the file system is
71 | * ready for interaction.
72 | * @param {Function} listener The listener function.
73 | */
74 | function registerOnReady(listener) {
75 | readyListeners.push(listener);
76 | }
77 |
78 | /**
79 | * Gets the internal file system URL for a stored file
80 | * @param {string} filePath The original URL of the asset.
81 | * @param {Function} callbackSuccess Callback for successful path retrieval.
82 | * @param {Function} callbackFail Callback for failed path retrieval.
83 | */
84 | function getPath(filePath, callbackSuccess, callbackFail) {
85 | if (ready) {
86 |
87 | filePath = RAL.Sanitiser.cleanURL(filePath);
88 |
89 | root.getFile(filePath, {}, function(fileEntry) {
90 | callbackSuccess(fileEntry.toURL());
91 | }, callbackFail);
92 | }
93 | }
94 |
95 | /**
96 | * Gets the file data as text for a stored file
97 | * @param {string} filePath The original URL of the asset.
98 | * @param {Function} callbackSuccess Callback for successful path retrieval.
99 | * @param {Function} callbackFail Callback for failed path retrieval.
100 | */
101 | function getDataAsText(filePath, callbackSuccess, callbackFail) {
102 |
103 | if (ready) {
104 |
105 | filePath = RAL.Sanitiser.cleanURL(filePath);
106 |
107 | root.getFile(filePath, {}, function(fileEntry) {
108 |
109 | fileEntry.file(function(file) {
110 | var reader = new FileReader();
111 | reader.onloadend = function(evt) {
112 | callbackSuccess(this.result);
113 | };
114 |
115 | reader.readAsText(file);
116 | });
117 |
118 | }, callbackFail);
119 | }
120 | }
121 |
122 | /**
123 | * Puts the file data in the file system.
124 | * @param {string} filePath The original URL of the asset.
125 | * @param {Blob} fileData The file data blob to store.
126 | * @param {Function} callback Callback for file storage.
127 | */
128 | function set(filePath, fileData, callback) {
129 |
130 | if(ready) {
131 |
132 | filePath = RAL.Sanitiser.cleanURL(filePath);
133 |
134 | var dirPath = filePath.split("/");
135 | dirPath.pop();
136 |
137 | // create the directories all the way
138 | // down to the path
139 | createDir(root, dirPath, function() {
140 |
141 | // now get a reference to our file, create it
142 | // if necessary
143 | root.getFile(filePath, {create: true}, function(fileEntry) {
144 |
145 | // create a writer on the file reference
146 | fileEntry.createWriter(function(fileWriter) {
147 |
148 | // catch on file ends
149 | fileWriter.onwriteend = function(e) {
150 |
151 | // update the writeend so when we have
152 | // truncated the file data we call the callback
153 | fileWriter.onwriteend = function(e) {
154 | callback(fileEntry.toURL());
155 | };
156 |
157 | // now truncate the file contents
158 | // for when we overwrite with a smaller file
159 | fileWriter.truncate(fileData.size);
160 | };
161 |
162 | // warn on write fails but right now don't bail
163 | fileWriter.onerror = function(e) {
164 | console.warn('Write failed: ' + e.toString());
165 | };
166 |
167 | // start writing
168 | fileWriter.write(fileData);
169 |
170 | }, callbacks.onError);
171 |
172 | }, callbacks.onError);
173 | });
174 | }
175 | }
176 |
177 | /**
178 | * Recursively creates the directories in a path.
179 | * @param {DirectoryEntry} rootDirEntry The base directory for this call.
180 | * @param {Array.
43 |
44 |
45 |
46 |
47 |
50 |
51 |
52 | Warnings
48 | 49 |
53 |