├── README.md
├── conf
    ├── amazon.urls
    ├── base.urls
    └── nginx.conf
├── lib
    └── resty
    │   └── s3.lua
└── lua-resty-s3-v1.0-1.rockspec


/README.md:
--------------------------------------------------------------------------------
  1 | Name
  2 | ====
  3 | 
  4 | lua-resty-s3 - upload content to amazon s3 with openresty
  5 | 
  6 | Table of Contents
  7 | =================
  8 | 
  9 | * [Name](#name)
 10 | * [Status](#status)
 11 | * [Description](#description)
 12 | * [Synopsis](#synopsis)
 13 | * [Methods](#methods)
 14 |     * [new](#new)
 15 |     * [generate_auth_headers](#generate_auth_headers)
 16 |     * [try_upload](#try_upload)
 17 |     * [upload_url](#upload_url)
 18 |     * [extract_urls](#extract_urls)
 19 |     * [upload_content](#upload_content)
 20 | * [Limitations](#limitations)
 21 | * [Installation](#installation)
 22 | * [TODO](#todo)
 23 | * [Author](#author)
 24 | * [Copyright and License](#copyright-and-license)
 25 | * [See Also](#see-also)
 26 | 
 27 | Status
 28 | ======
 29 | 
 30 | This library is still under early development and considered experimental.
 31 | 
 32 | Description
 33 | ===========
 34 | 
 35 | This Lua library is a s3 uploading utility for the ngx_lua nginx module:
 36 | 
 37 | Synopsis
 38 | ========
 39 | 
 40 | ```
 41 |     lua_package_path "/path/to/lua-resty-s3/lib/?.lua;;";
 42 |     
 43 |     server {
 44 |         location /test {
 45 |             content_by_lua '
 46 |                 local s3 = require "resty.s3"
 47 |                 local s3, err = s3:new("aws-id", "aws-key")
 48 |                 
 49 |                 final_url, err = s3:upload_url("http://lorempixel.com/400/200/", "examplebucket", "lorempixel400x200")`
 50 |             ';
 51 |         }
 52 |         
 53 |         include conf/*.urls;
 54 |     }
 55 | ```
 56 | 
 57 | [Back to TOC](#table-of-contents)
 58 | 
 59 | Methods
 60 | =======
 61 | 
 62 | All of the commands return either something that evaluates to true on success, or `nil` and an error message on failure.
 63 | 
 64 | new
 65 | ---
 66 | `syntax: s3, err = s3:new(id, key)`
 67 | 
 68 | Creates an uploading object. In case of failures, returns `nil` and a string describing the error.
 69 | 
 70 | [Back to TOC](#table-of-contents)
 71 | 
 72 | generate_auth_headers
 73 | ---------------------
 74 | `syntax: s3, err = s3:generate_auth_headers(content_type, destination)`
 75 | 
 76 | `syntax: s3, err = s3:generate_auth_headers("binary/octet-stream", "/examplebucket/lorempixel400x200")`
 77 | 
 78 | Creates the headers needed for authentication with amazon. In case of failures, returns `nil` and a string describing the error.
 79 | 
 80 | [Back to TOC](#table-of-contents)
 81 | 
 82 | try_upload
 83 | ----------
 84 | `syntax: s3, err = s3:try_upload(content, destination, content_type, headers)`
 85 | 
 86 | `syntax: s3, err = s3:try_upload([[<h1>Hello</h1>]], "/examplebucket/hello", "text/html", headers)`
 87 | 
 88 | Attempts to upload content to s3. In case of failures, returns `nil` and a string describing the error.
 89 | 
 90 | [Back to TOC](#table-of-contents)
 91 | 
 92 | upload_url
 93 | ----------
 94 | `syntax: final_url, err = s3:upload_url(file_url, bucket, object_name, check_for_existance, add_to_existance)`
 95 | 
 96 | `syntax: final_url, err = s3:upload_url("http://lorempixel.com/400/200/", "/examplebucket/", "lorempixel400x200")`
 97 | 
 98 | Attempts to upload content to s3 from the url set by file_url and the id/key set with new(). If object_name is supplied then that will be the name of the new file, otherwise it will hash the file_url to create a unique key for it.
 99 | 
100 | Callbacks for checking something before uploading [again], and after uploading can be supplied in check_for_existance and add_to_existance. Each will be called with the object_name or a hash.
101 | 
102 | ```
103 | local uploaded_content = ngx.shared.uploaded_content
104 | 
105 | check = function (name)
106 |   ok, err = uploaded_content:get(name)
107 |   if ok then return true end
108 | end
109 | 
110 | add = function (name)
111 |   ok, err = uploaded_content:add(name)
112 |   return true
113 | end
114 | 
115 | final_url, err = s3:upload_url("http://lorempixel.com/400/200/", "/examplebucket/", "lorempixel400x200", check, add)
116 | ```
117 | 
118 | In case of success, returns the new url. In case of errors, returns `nil` with a string describing the error.
119 | 
120 | [Back to TOC](#table-of-contents)
121 | 
122 | extract_urls
123 | ------------
124 | `syntax: s3, err = s3:extract_urls(file_content, bucket)`
125 | 
126 | `syntax: s3, err = s3:extract_urls([[<ahref="www.google.com"><img src="http://lorempixel.com/output/city-q-g-640-480-4.jpg" /></a>]], "/examplebucket/")`
127 | 
128 | Attempts to find and upload urls from within source. In case of failures, returns `nil` and a string describing the error.
129 | 
130 | [Back to TOC](#table-of-contents)
131 | 
132 | upload_content
133 | --------------
134 | `syntax: s3, err = s3:upload_content(file_content, bucket, object_name, check_for_existance, add_to_existance)`
135 | 
136 | `syntax: s3, err = s3:upload_content([[<ahref="www.google.com"><img src="http://lorempixel.com/output/city-q-g-640-480-4.jpg" /></a>]], "/examplebucket/", "city.jpg")`
137 | 
138 | Attempts to upload content to s3 (handles all auth automatically). In case of failures, returns `nil` and a string describing the error.
139 | 
140 | [Back to TOC](#table-of-contents)
141 | 
142 | Limitations
143 | ===========
144 | 
145 | 
146 | 
147 | [Back to TOC](#table-of-contents)
148 | 
149 | Installation
150 | ============
151 | You can install it with luarocks `luarocks install lua-resty-s3`
152 | 
153 | Otherwise you need to configure the lua_package_path directive to add the path of your lua-nginx-loggin source to ngx_lua's LUA_PATH search path, as in
154 | 
155 | ```nginx
156 |     # nginx.conf
157 |     http {
158 |         lua_package_path "/path/to/lua-resty-s3/lib/?.lua;;";
159 |         ...
160 |     }
161 | ```
162 | 
163 | This package also requires the luasocket and xxhash packages to be installed http://w3.impa.br/~diego/software/luasocket/ , https://github.com/mah0x211/lua-xxhash
164 | ```
165 | luarocks install luasocket
166 | ```
167 | 
168 | Ensure that the system account running your Nginx ''worker'' proceses have
169 | enough permission to read the `.lua` file.
170 | 
171 | [Back to TOC](#table-of-contents)
172 | 
173 | TODO
174 | ====
175 | 
176 | 
177 | 
178 | [Back to TOC](#table-of-contents)
179 | 
180 | Author
181 | ======
182 | 
183 | James Marlowe "jamesmarlowe" <jameskmarlowe@gmail.com>, Lumate LLC.
184 | 
185 | [Back to TOC](#table-of-contents)
186 | 
187 | Copyright and License
188 | =====================
189 | 
190 | This module is licensed under the BSD license.
191 | 
192 | Copyright (C) 2012-2014, by James Marlowe (jamesmarlowe) <jameskmarlowe@gmail.com>, Lumate LLC.
193 | 
194 | All rights reserved.
195 | 
196 | Redistribution and use in source and binary forms, with or without
197 | modification, are permitted provided that the following conditions are met:
198 | 
199 | * Redistributions of source code must retain the above copyright notice, this
200 |   list of conditions and the following disclaimer.
201 | 
202 | * Redistributions in binary form must reproduce the above copyright notice,
203 |   this list of conditions and the following disclaimer in the documentation
204 |   and/or other materials provided with the distribution.
205 | 
206 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
207 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
208 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
209 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
210 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
211 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
212 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
213 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
214 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
215 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
216 | 
217 | [Back to TOC](#table-of-contents)
218 | 
219 | See Also
220 | ========
221 | * the ngx_lua module: http://wiki.nginx.org/HttpLuaModule
222 | * the [lua-resty-hmac](https://github.com/jamesmarlowe/lua-resty-hmac) library
223 | 
224 | [Back to TOC](#table-of-contents)
225 | 


--------------------------------------------------------------------------------
/conf/amazon.urls:
--------------------------------------------------------------------------------
 1 | 
 2 |         location /lua-resty-s3/upload/ {
 3 |             internal;
 4 |             resolver 8.8.8.8;
 5 |             set_unescape_uri $date $arg_date;
 6 |             set_unescape_uri $auth $arg_auth;
 7 |             set_unescape_uri $file $arg_file;
 8 |             set_unescape_uri $mime $arg_mime;
 9 | 
10 |             proxy_pass_request_headers off;
11 |             more_clear_headers 'Host';
12 |             more_clear_headers 'Connection';
13 |             more_clear_headers 'Content-Length';
14 |             more_clear_headers 'User-Agent';
15 |             more_clear_headers 'Accept';
16 | 
17 |             proxy_set_header Date $date;
18 |             proxy_set_header Authorization $auth;
19 |             proxy_set_header content-type $mime;
20 |             proxy_set_header Content-MD5 '';
21 | 
22 |             proxy_pass http://s3.amazonaws.com:80$file;
23 |         }
24 | 


--------------------------------------------------------------------------------
/conf/base.urls:
--------------------------------------------------------------------------------
 1 | 
 2 |         location = /lua-resty-s3/proxy/ {
 3 |             internal;
 4 |             
 5 |             set_unescape_uri $my_host $arg_host;
 6 |             set_unescape_uri $my_uri $arg_uri;
 7 |             
 8 |             resolver 8.8.8.8;
 9 |             
10 |             proxy_set_header User-Agent 'Mozilla/5.0 (X11; Linux x86_64; rv:16.0)';
11 |             proxy_pass http://$my_host:80$my_uri;
12 |         }
13 |         
14 |         location = /lua-resty-s3/test/ {
15 |             internal;
16 |             
17 |             content_by_lua '
18 |                 local s3 = require "s3"
19 |                 
20 |                 local params = ngx.req.get_query_args()
21 |                 
22 |                 if not params.aws_id or not params.aws_key then
23 |                     ngx.say("no aws id or key")
24 |                     ngx.exit(ngx.OK)
25 |                 end
26 |                 
27 |                 local ms3, err = s3:new(params.aws_id, params.aws_key)
28 |                 
29 |                 if err then ngx.say(err) ngx.exit(ngx.OK) end
30 | 
31 |                 local final_url, err = ms3:upload_url("http://lorempixel.com/400/200/", params.aws_bucket, "lorempixel400x200")
32 |                 
33 |                 if err then
34 |                     ngx.say(err)
35 |                 else
36 |                     ngx.say("File uploaded at: "..final_url)
37 |                 end
38 |                 
39 |                 
40 |                 local final_url, err = ms3:upload_content([[<img src="http://lorempixel.com/400/200/city/" />]], params.aws_bucket, "city")
41 | 
42 |                 if err then
43 |                     ngx.say(err)
44 |                 else
45 |                     ngx.say("File uploaded at: "..final_url)
46 |                 end
47 |             ';
48 |         }
49 | 


--------------------------------------------------------------------------------
/conf/nginx.conf:
--------------------------------------------------------------------------------
 1 | http {
 2 |     
 3 |     lua_package_path "${prefix}/lib/?.lua;;";
 4 |     
 5 |     server {
 6 |     
 7 |         location /test {
 8 |             content_by_lua '
 9 |                 local s3 = require "resty.s3"
10 |                 local s3, err = s3:new("aws-id", "aws-key")
11 |                 
12 |                 final_url, err = s3:upload_url("http://lorempixel.com/400/200/", "examplebucket", "lorempixel400x200")`
13 |             ';
14 |         }
15 |     
16 |         include *.urls;
17 |         
18 |     }
19 | }
20 | 


--------------------------------------------------------------------------------
/lib/resty/s3.lua:
--------------------------------------------------------------------------------
  1 | -- Copyright (C) James Marlowe (jamesmarlowe), Lumate LLC.
  2 | 
  3 | 
  4 | local xxhash = require "xxhash"
  5 | local url = require "socket.url"
  6 | local hmac = require "resty.hmac"
  7 | local hash_seed = 0x1db1e298
  8 | local upload_url = "/lua-resty-s3/upload/"
  9 | local proxy_url = "/lua-resty-s3/proxy/"
 10 | 
 11 | 
 12 | local ok, new_tab = pcall(require, "table.new")
 13 | if not ok then
 14 |     new_tab = function (narr, nrec) return {} end
 15 | end
 16 | 
 17 | 
 18 | local _M = new_tab(0, 155)
 19 | _M._VERSION = '0.01'
 20 | 
 21 | 
 22 | local mt = { __index = _M }
 23 | 
 24 | 
 25 | function _M.new(self, id, key)
 26 |     local id, key = id, key
 27 |     
 28 |     if not id then
 29 |         return nil, "must provide id"
 30 |     end
 31 |     if not key then
 32 |         return nil, "must provide key"
 33 |     end
 34 |     
 35 |     return setmetatable({ id = id, key = key }, mt)
 36 | end
 37 | 
 38 | 
 39 | function _M.generate_auth_headers(self, content_type, destination)
 40 |     local id, key = self.id, self.key
 41 |     
 42 |     if not id or not key then
 43 |         return nil, "not initialized"
 44 |     end
 45 |     
 46 |     local date = os.date("%a, %d %b %Y %H:%M:%S +0000")
 47 |     local hm, err = hmac:new(key)
 48 |     local StringToSign = "PUT"..string.char(10)..string.char(10)..content_type..string.char(10)..date..string.char(10)..destination
 49 |     headers, err = hm:generate_headers("AWS", id, "sha1", StringToSign)
 50 |     
 51 |     return headers, err
 52 | end
 53 | 
 54 | 
 55 | function _M.try_upload(self, content, destination, content_type, headers)
 56 |     local id, key = self.id, self.key
 57 |     
 58 |     if not id or not key then
 59 |         return nil, "not initialized"
 60 |     end
 61 |     
 62 |     local retry = 0
 63 |     while (not resp or resp.status ~= 200) and retry < 3 do
 64 |         resp = ngx.location.capture(
 65 |               upload_url,
 66 |             { method = ngx.HTTP_PUT,
 67 |               body = content,
 68 |               args = {date=headers.date, auth=headers.auth, file=destination, mime=content_type}}
 69 |         )
 70 |         retry = retry + 1
 71 |     end
 72 |     
 73 |     return resp
 74 | end
 75 | 
 76 | 
 77 | function _M.upload_url(self, file_url, bucket, object_name, check_for_existance, add_to_existance)
 78 |     local id, key = self.id, self.key
 79 |     
 80 |     if not id or not key then
 81 |         return nil, "not initialized"
 82 |     end
 83 |     
 84 |     if not file_url then
 85 |         return nil, "nothing to upload"
 86 |     end
 87 |     
 88 |     if not bucket then
 89 |         return nil, "unknown bucket"
 90 |     end
 91 |     
 92 |     if not object_name then object_name = xxhash.xxh32(file_url, hash_seed) end
 93 |     
 94 |     local destination = bucket..object_name
 95 |     local s3_url = "http://s3.amazonaws.com"
 96 |     local final_url = s3_url..destination
 97 |     local content_type = "binary/octet-stream"
 98 |     
 99 |     if check_for_existance and check_for_existance(object_name) then
100 |         return final_url
101 |     end
102 |     
103 |     file_url = file_url:gsub([["]],"")
104 |     file_url = url.parse(file_url)
105 |     file_content = ngx.location.capture(proxy_url, {args={host=file_url.host,uri=file_url.path}})
106 |     
107 |     if file_content.status == 200 then 
108 |         local headers, err = self:generate_auth_headers(content_type, destination)
109 |         if not headers then return nil, err end
110 |         
111 |         local res = self:try_upload(file_content.body, destination, content_type, headers)
112 | 
113 |         if res.status == 200 then
114 |             if add_to_existance then
115 |                 add_to_existance(object_name)
116 |             end
117 |             
118 |             return final_url
119 |         else
120 |             return nil, "Could not upload: "..resp.status.." "..resp.body
121 |         end
122 |     else
123 |         return nil, "could not get url: "..url.build(file_url)
124 |     end
125 | end
126 | 
127 | 
128 | function _M.extract_urls(self, file_content, bucket)
129 |     pos = 1
130 |     for st,sp in function() return file_content:find([["]],pos, true) end do
131 |         sp = file_content:find([["]],st+1, true)
132 |         embedded_url = file_content:sub(st, sp)
133 |         
134 |         if embedded_url:find("//", 1, true) then
135 |             embedded_url = embedded_url:sub(0,embedded_url:find("?", 1, true))
136 |             
137 |             if embedded_url:find(".", -6, true) then
138 |                 object_name = xxhash.xxh32(embedded_url, hash_seed)
139 |                 new_url, err = self:upload_url(embedded_url, bucket, object_name, check_for_existance, add_to_existance)
140 |                 
141 |                 if not new_url then 
142 |                     file_content = file_content:sub(0,st)..file_content:sub(sp)
143 |                     sp = st+1
144 |                     
145 |                 else
146 |                     file_content = file_content:sub(1,st)..new_url..[["]]..file_content:sub(sp)
147 |                     sp= st+#new_url+1
148 |                     
149 |                 end
150 |             else
151 |                 file_content = file_content:sub(0,st)..file_content:sub(sp)
152 |                 sp = st+1
153 |                 
154 |             end
155 |         end
156 |         pos = (sp or (#file_content -1)) + 1
157 |     end
158 |     
159 |     return file_content
160 | end
161 | 
162 | 
163 | function _M.upload_content(self, file_content, bucket, object_name, check_for_existance, add_to_existance)
164 |     local id, key = self.id, self.key
165 |     
166 |     if not id or not key then
167 |         return nil, "not initialized"
168 |     end
169 |     
170 |     if not file_content then
171 |         return nil, "nothing to upload"
172 |     end
173 |     
174 |     if not bucket then
175 |         return nil, "unknown bucket"
176 |     end
177 |     
178 |     if not object_name then object_name = xxhash.xxh32(file_content, hash_seed) end
179 |     
180 |     local destination = bucket..object_name
181 |     local s3_url = "http://s3.amazonaws.com"
182 |     local final_url = s3_url..destination
183 |     local content_type = "text/html"
184 |     
185 |     if check_for_existance and check_for_existance(object_name) then
186 |         return final_url
187 |     end
188 | 
189 |     file_content = self:extract_urls(file_content, bucket)
190 |     
191 |     headers, err = self:generate_auth_headers(content_type, destination)
192 |     if not headers then return nil, err end
193 |     
194 |     local resp = self:try_upload(file_content, destination, content_type, headers)
195 |     
196 |     if resp.status == 200 then
197 |         if add_to_existance then
198 |             add_to_existance(object_name)
199 |         end
200 |         return final_url
201 |     else
202 |         return nil, "Could not upload: "..resp.status.." "..resp.body
203 |     end
204 | end
205 | 
206 | 
207 | 
208 | return _M
209 | 


--------------------------------------------------------------------------------
/lua-resty-s3-v1.0-1.rockspec:
--------------------------------------------------------------------------------
 1 | package = "lua-resty-s3"
 2 | version = "v1.0-1"
 3 | 
 4 | source = {
 5 |   url = "git://github.com/jamesmarlowe/lua-resty-s3.git"
 6 | }
 7 | 
 8 | description = {
 9 |   summary = "Upload content to amazon s3 with OpenResty",
10 |   homepage = "https://github.com/jamesmarlowe/lua-resty-s3",
11 |   license = "BSD",
12 |   maintainer = "jameskmarlowe@gmail.com"
13 | }
14 | 
15 | dependencies = {
16 |   "lua >= 5.1",
17 |   "lua-resty-hmac",
18 |   "luasocket",
19 |   "xxhash"
20 | }
21 | 
22 | build = {
23 |     type = "builtin",
24 |     modules = {
25 |         ["resty.s3"] = "lib/resty/s3.lua"
26 |     }
27 | }
28 | 


--------------------------------------------------------------------------------