├── cloud_storage ├── http.moon ├── http.lua ├── multipart.moon ├── oauth.moon ├── mock.moon ├── oauth.lua ├── multipart.lua ├── mock.lua ├── google.moon └── google.lua ├── Makefile ├── cloud_storage-dev-1.rockspec ├── .github └── workflows │ └── test.yml ├── spec ├── test_key.pem ├── test_key.json └── cloud_storage_spec.moon ├── CLAUDE.md ├── TODO.md ├── tags └── README.md /cloud_storage/http.moon: -------------------------------------------------------------------------------- 1 | 2 | -- implementation agnostic access http.request 3 | -- require"cloud_storage.http".set request: -> print "hello!" 4 | 5 | local _http 6 | 7 | default = -> require "socket.http" 8 | 9 | get = -> 10 | _http = default! unless _http 11 | _http 12 | 13 | set = (http) -> 14 | _http = http 15 | 16 | { :get, :set } 17 | -------------------------------------------------------------------------------- /cloud_storage/http.lua: -------------------------------------------------------------------------------- 1 | local _http 2 | local default 3 | default = function() 4 | return require("socket.http") 5 | end 6 | local get 7 | get = function() 8 | if not (_http) then 9 | _http = default() 10 | end 11 | return _http 12 | end 13 | local set 14 | set = function(http) 15 | _http = http 16 | end 17 | return { 18 | get = get, 19 | set = set 20 | } 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build local 2 | 3 | build: 4 | moonc cloud_storage/ 5 | 6 | local: build 7 | luarocks --lua-version=5.1 make --local cloud_storage-dev-1.rockspec 8 | 9 | %.pem: %.p12 10 | openssl pkcs12 -in $< -out $@ -nodes -clcerts 11 | 12 | 13 | %.rsa.pem: %.p12 14 | openssl pkcs12 -nodes -nocerts -in $< | openssl rsa -out $@ 15 | 16 | tags:: 17 | moon-tags $$(git ls-files cloud_storage/ | grep '\.moon$$') > tags 18 | -------------------------------------------------------------------------------- /cloud_storage-dev-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "cloud_storage" 2 | version = "dev-1" 3 | 4 | source = { 5 | url = "git://github.com/leafo/cloud_storage.git", 6 | } 7 | 8 | description = { 9 | summary = "Access Google Cloud Storage from Lua", 10 | license = "MIT", 11 | maintainer = "Leaf Corcoran ", 12 | } 13 | 14 | dependencies = { 15 | "lua >= 5.1", 16 | "luasocket", 17 | "lua-cjson", 18 | "mimetypes", 19 | "luaossl", 20 | "date", 21 | "luaexpat", 22 | } 23 | 24 | build = { 25 | type = "builtin", 26 | modules = { 27 | ["cloud_storage.mock"] = "cloud_storage/mock.lua", 28 | ["cloud_storage.google"] = "cloud_storage/google.lua", 29 | ["cloud_storage.oauth"] = "cloud_storage/oauth.lua", 30 | ["cloud_storage.http"] = "cloud_storage/http.lua", 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: [push] 4 | 5 | jobs: 6 | test: 7 | strategy: 8 | fail-fast: false 9 | matrix: 10 | luaVersion: ["5.1", "5.2", "5.3", "5.4", "luajit-openresty"] 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@master 16 | 17 | - uses: leafo/gh-actions-lua@master 18 | with: 19 | luaVersion: ${{ matrix.luaVersion }} 20 | 21 | - uses: leafo/gh-actions-luarocks@master 22 | 23 | - name: build 24 | run: | 25 | luarocks install busted 26 | luarocks install moonscript 27 | luarocks install https://raw.githubusercontent.com/leafo/lua-cjson/master/lua-cjson-dev-1.rockspec 28 | luarocks make 29 | 30 | - name: test 31 | run: | 32 | busted -o utfTerminal 33 | 34 | -------------------------------------------------------------------------------- /spec/test_key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIICXQIBAAKBgQC03qh7XLNCpfyEq6wwzKQJwNRGvbL82s5gw+MBwp9nriSA9Hwr 3 | KQIbZc8BHqaTuK1no2kDdbjMMxopKvE27q9qh9LOz63ZiwRr+LNt86QcZIeF6tNX 4 | quMr00rIxBaR6sghkdPB7m+1P1LMgx+zFYahB9HHuae5blTI0deTf0AI/QIDAQAB 5 | AoGAcFXaTsREkiCFteDqEWUIfQZG0akAggtkIrWHSJCYcMy331/5vtS5ekrBRvDC 6 | hP0uti/ICV4UaL9UgD0rk/Kq/3QLKyBz27GWM07tls48jXedJCj0ZkkZh17pgl5f 7 | vred6aaexoJszuIL4QNvn5X9ynQXqlh/fkKq0Z91VnZ/oAECQQDaBorx2eLbm9jG 8 | xVUKUdv+jXEdCLh1AC6Vgh1c94smhtv66escZA54J0PHABv0AjnW09s6Y17KRmQq 9 | ir8tsFlZAkEA1F9jyx4M/ZXTxD0n1KvWnHN3YcNH4VwK+XOHh4Kq2JprzMwr8eyM 10 | hWRFBBGvPx5c0JN33f9heFN/jrPv0QUURQJAbbWxITY087EeihcuTb0XaKYf7y4+ 11 | M5Hd3xnUUL235bEi7MXcqzKmHUwUzQR/DHA6TqHYxS7PuhVlvgqHXSRKMQJBALfd 12 | rmSYq96Q7TslR2rVK4VgYqd9jqoKKmY2I8yq0Iefil1RF2roxfBnE2mmdfdLrkfW 13 | pRzKkfS/Ndyy5Jour5ECQQCRwhwda//rHm1bsw33pzI08sA9sy/jCG4zDy6u1gjI 14 | 5SubD/VaOpaivqqJLbEVwO3W8MxlGCxaA2iyZpuw5M9N 15 | -----END RSA PRIVATE KEY----- 16 | -------------------------------------------------------------------------------- /spec/test_key.json: -------------------------------------------------------------------------------- 1 | {"auth_uri":"https:\/\/accounts.google.com\/o\/oauth2\/auth","type":"service_account","client_x509_cert_url":"https:\/\/www.googleapis.com\/robot\/v1\/metadata\/x509\/leaf%40leafo.net","private_key_id":"helloworld","auth_provider_x509_cert_url":"https:\/\/www.googleapis.com\/oauth2\/v1\/certs","token_uri":"https:\/\/accounts.google.com\/o\/oauth2\/token","private_key":"-----BEGIN RSA PRIVATE KEY-----\nMIICXQIBAAKBgQC03qh7XLNCpfyEq6wwzKQJwNRGvbL82s5gw+MBwp9nriSA9Hwr\nKQIbZc8BHqaTuK1no2kDdbjMMxopKvE27q9qh9LOz63ZiwRr+LNt86QcZIeF6tNX\nquMr00rIxBaR6sghkdPB7m+1P1LMgx+zFYahB9HHuae5blTI0deTf0AI\/QIDAQAB\nAoGAcFXaTsREkiCFteDqEWUIfQZG0akAggtkIrWHSJCYcMy331\/5vtS5ekrBRvDC\nhP0uti\/ICV4UaL9UgD0rk\/Kq\/3QLKyBz27GWM07tls48jXedJCj0ZkkZh17pgl5f\nvred6aaexoJszuIL4QNvn5X9ynQXqlh\/fkKq0Z91VnZ\/oAECQQDaBorx2eLbm9jG\nxVUKUdv+jXEdCLh1AC6Vgh1c94smhtv66escZA54J0PHABv0AjnW09s6Y17KRmQq\nir8tsFlZAkEA1F9jyx4M\/ZXTxD0n1KvWnHN3YcNH4VwK+XOHh4Kq2JprzMwr8eyM\nhWRFBBGvPx5c0JN33f9heFN\/jrPv0QUURQJAbbWxITY087EeihcuTb0XaKYf7y4+\nM5Hd3xnUUL235bEi7MXcqzKmHUwUzQR\/DHA6TqHYxS7PuhVlvgqHXSRKMQJBALfd\nrmSYq96Q7TslR2rVK4VgYqd9jqoKKmY2I8yq0Iefil1RF2roxfBnE2mmdfdLrkfW\npRzKkfS\/Ndyy5Jour5ECQQCRwhwda\/\/rHm1bsw33pzI08sA9sy\/jCG4zDy6u1gjI\n5SubD\/VaOpaivqqJLbEVwO3W8MxlGCxaA2iyZpuw5M9N\n-----END RSA PRIVATE KEY-----\n","project_id":"cloud_storage_test","client_email":"leaf@leafo.net","client_id":"111111111111"} 2 | -------------------------------------------------------------------------------- /cloud_storage/multipart.moon: -------------------------------------------------------------------------------- 1 | 2 | mimetypes = require "mimetypes" 3 | url = require "socket.url" 4 | 5 | import insert, concat from table 6 | 7 | math.randomseed os.time! 8 | 9 | import type from require "moon" 10 | 11 | class File 12 | new: (@fname) => 13 | mime: => mimetypes.guess @fname 14 | content: => 15 | if file = io.open @fname 16 | with file\read "*a" 17 | file\close! 18 | 19 | rand_string = (len) -> 20 | shuffled = for i=1,len 21 | r = math.random 97, 122 22 | r-= 32 if math.random! >= 0.5 23 | r 24 | string.char unpack shuffled 25 | 26 | -- multipart encodes params 27 | -- returns encoded string,boundary 28 | -- params is an a table of tuple tables: 29 | -- params = { 30 | -- {key1, value2}, 31 | -- {key2, value2}, 32 | -- } 33 | encode = (params) -> 34 | chunks = for tuple in *params 35 | k,v = unpack tuple 36 | 37 | k = url.escape k 38 | buffer = { 'Content-Disposition: form-data; name="'.. k .. '"' } 39 | 40 | content = if type(v) == File 41 | -- how is this encoded? 42 | buffer[1] ..= '; filename="' .. v.fname .. '"' 43 | insert buffer, "Content-type: #{v\mime!}" 44 | v\content! 45 | else 46 | v 47 | 48 | insert buffer, "" 49 | insert buffer, content 50 | concat buffer, "\r\n" 51 | 52 | local boundary 53 | while true 54 | boundary = "Boundary#{rand_string 16}" 55 | for c in *chunks 56 | continue if c\find boundary 57 | do break 58 | 59 | inner = concat { "\r\n", "--", boundary, "\r\n" } 60 | 61 | (concat { 62 | "--", boundary, "\r\n" 63 | concat chunks, inner 64 | "\r\n", "--", boundary, "--", "\r\n" 65 | }), boundary 66 | 67 | encode_tbl = (params) -> 68 | encode [{k,v} for k,v in pairs params] 69 | 70 | { :encode, :encode_tbl, :File } 71 | -------------------------------------------------------------------------------- /cloud_storage/oauth.moon: -------------------------------------------------------------------------------- 1 | url = require "socket.url" 2 | mime = require "mime" 3 | json = require "cjson" 4 | 5 | pkey = require "openssl.pkey" 6 | digest = require "openssl.digest" 7 | 8 | h = require"cloud_storage.http" 9 | 10 | param = (tbl) -> 11 | tuples = for k,v in pairs tbl 12 | "#{url.escape k}=#{url.escape v}" 13 | 14 | table.concat tuples, "&" 15 | 16 | class OAuth 17 | auth_url: "https://accounts.google.com/o/oauth2/token" 18 | header: '{"alg":"RS256","typ":"JWT"}' 19 | digest_type: "sha256WithRSAEncryption" 20 | 21 | scope: { 22 | read_only: "https://www.googleapis.com/auth/devstorage.read_only" 23 | read_write: "https://www.googleapis.com/auth/devstorage.read_write" 24 | full_control: "https://www.googleapis.com/auth/devstorage.full_control" 25 | } 26 | 27 | new: (@client_email, @private_key_file) => 28 | 29 | get_access_token: => 30 | if not @access_token or os.time! >= @expires_at 31 | @refresh_access_token! 32 | 33 | @access_token 34 | 35 | refresh_access_token: => 36 | http = h.get! 37 | 38 | time = os.time! 39 | jwt = @_make_jwt @client_email, @private_key 40 | 41 | req_params = param { 42 | grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer" 43 | assertion: jwt 44 | } 45 | 46 | res = assert http.request @auth_url, req_params 47 | res = json.decode res 48 | 49 | if res.error 50 | error "Failed auth: #{res.error}" 51 | 52 | @expires_at = time + res.expires_in 53 | @access_token = res.access_token 54 | @access_token 55 | 56 | sign_string: (string) => 57 | d = assert digest.new @digest_type 58 | key = @_private_key! 59 | d\update string 60 | (mime.b64 assert key\sign d) 61 | 62 | _load_private_key: (str) => 63 | with key = assert pkey.new str 64 | @_private_key = -> key 65 | 66 | _private_key: => 67 | @_load_private_key assert assert(io.open(@private_key_file))\read "*a" 68 | 69 | _make_jwt: (client_email, private_key) => 70 | hr = 60*60 71 | claims = json.encode { 72 | iss: client_email 73 | aud: @auth_url 74 | scope: @scope.full_control 75 | iat: os.time! 76 | exp: os.time! + hr 77 | } 78 | 79 | sig_input = mime.b64(@header) .. "." .. mime.b64(claims) 80 | signature = @sign_string sig_input 81 | 82 | sig_input .. "." .. signature 83 | 84 | { :OAuth } 85 | 86 | -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | # CLAUDE.md 2 | 3 | This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. 4 | 5 | ## Development Commands 6 | 7 | ### Build 8 | - `make build` or `moonc cloud_storage/` - Compile MoonScript files to Lua 9 | - `make local` - Build and install locally with LuaRocks (lua 5.1) 10 | 11 | ### Linting 12 | - `moonc -l FILE...` - Lint MoonScript files to verify syntax and coding standards 13 | 14 | ### Testing 15 | - `busted` - Run all tests using the Busted framework 16 | - `busted spec/cloud_storage_spec.moon` - Run specific test file 17 | 18 | ### Package Management 19 | - `luarocks make --local cloud_storage-dev-1.rockspec` - Install locally for development 20 | - `luarocks pack cloud_storage-dev-1.rockspec` - Create source rock package 21 | 22 | ## Project Architecture 23 | 24 | This is a Lua library written in MoonScript for accessing Google Cloud Storage and Cloudflare R2. The codebase follows a modular design: 25 | 26 | ### MoonScript Development 27 | - Source files are written in MoonScript (`.moon` files) 28 | - ONLY edit `.moon` files - never edit the generated `.lua` files directly 29 | - Both `.moon` and `.lua` files are checked into the repo for portability 30 | - Use `moonc` to compile MoonScript to Lua 31 | - Always lint with `moonc -l` before committing changes 32 | 33 | ### Core Modules 34 | - `cloud_storage/init.moon` - Main module entry point with version info 35 | - `cloud_storage/google.moon` - Google Cloud Storage API implementation 36 | - `cloud_storage/oauth.moon` - OAuth2 authentication for Google services 37 | - `cloud_storage/r2.moon` - Cloudflare R2 compatible storage interface 38 | - `cloud_storage/http.moon` - HTTP client abstraction layer 39 | - `cloud_storage/multipart.moon` - Multipart upload functionality 40 | - `cloud_storage/mock.moon` - Mock implementation for testing 41 | 42 | ### Authentication 43 | The library supports two authentication methods: 44 | 1. JSON service account keys (recommended) 45 | 2. P12/PEM private key files with service account email 46 | 47 | ### HTTP Client 48 | Uses `socket.http` by default but supports custom HTTP clients through the `cloud_storage.http` module's `set()` function. 49 | 50 | ### Code Style 51 | - Written in MoonScript (`.moon` files) which compiles to Lua 52 | - Lua files are generated artifacts and should not be edited directly 53 | - Test files use Busted framework with MoonScript syntax 54 | - Follow existing patterns for error handling: return `nil, error_message, error_object` on failure 55 | - Use `moonc -l` to verify syntax and coding standards 56 | 57 | ### Dependencies 58 | Key runtime dependencies (from rockspec): 59 | - `luasocket` - HTTP client and networking 60 | - `lua-cjson` - JSON parsing 61 | - `luaossl` - Cryptographic operations 62 | - `date` - Date/time handling 63 | - `luaexpat` - XML parsing 64 | - `mimetypes` - MIME type detection 65 | -------------------------------------------------------------------------------- /cloud_storage/mock.moon: -------------------------------------------------------------------------------- 1 | 2 | import Bucket from require "cloud_storage.google" 3 | 4 | shell_escape = (str) -> 5 | str\gsub "'", "''" 6 | 7 | execute = (cmd) -> 8 | -- print "RUN: #{cmd}" 9 | proc = io.popen cmd 10 | out = proc\read "*a" 11 | proc\close! 12 | out\match "^(.-)%s*$" 13 | 14 | class MockStorage 15 | new: (@dir_name=".", @url_prefix="") => 16 | 17 | bucket: (bucket) => Bucket bucket, @ 18 | 19 | _full_path: (bucket, key) => 20 | dir = if @dir_name == "." then "" else @dir_name .. "/" 21 | "#{dir}#{bucket}/#{key}" 22 | 23 | file_url: (bucket, key) => 24 | prefix = if @url_prefix == "" then "" else @url_prefix .. "/" 25 | prefix .. @_full_path bucket, key 26 | 27 | get_service: => error "Not implemented" 28 | 29 | get_bucket: (bucket) => 30 | path = "#{@dir_name}/#{bucket}" 31 | execute "mkdir -p '#{shell_escape path}'" 32 | escaped_path = "$(echo '#{shell_escape path}' | sed -e 's/[\\/&]/\\\\&/g')" 33 | cmd = 'find "'..shell_escape(path)..'" -type f | sed -e "s/^'..escaped_path..'//"' 34 | files = execute cmd 35 | return for file in files\gmatch "[^\n]+" 36 | { key: file\match "/?(.*)" } 37 | 38 | put_file_string: (bucket, key, data, options={}) => 39 | assert not options.key, "key is not an option, but an argument" 40 | if type(data) == "table" 41 | error "put_file_string interface has changed: key is now the second argument" 42 | 43 | assert key, "missing key" 44 | assert type(data) == "string", "expected string for data" 45 | 46 | path = @_full_path bucket, key 47 | dir = execute "dirname '#{shell_escape path}'" 48 | 49 | execute "mkdir -p '#{shell_escape dir}'" 50 | with io.open path, "w" 51 | \write data 52 | \close! 53 | 200 54 | 55 | put_file: (bucket, fname, options={}) => 56 | data = if f = io.open fname 57 | with f\read "*a" 58 | f\close! 59 | else 60 | error "Failed to read file: #{fname}" 61 | 62 | @put_file_string bucket, data, options 63 | 64 | delete_file: (bucket, key) => 65 | path = @_full_path bucket, key 66 | os.execute "[ -a '#{shell_escape path}' ] && rm '#{shell_escape path}'" 67 | 200 68 | 69 | get_file: (bucket, key) => error "not implemented" 70 | head_file: (bucket, key) => error "Not implemented" 71 | 72 | 73 | if ... == "test" 74 | moon = require "moon" 75 | s = MockStorage("test_storage", "static") 76 | 77 | print s\_full_path "dad_bucket", "eat/my/sucks" 78 | print MockStorage!\_full_path "nobucket", "hello.world" 79 | 80 | print! 81 | 82 | b = s\bucket "my_bucket" 83 | 84 | b\put_file_string "this is a file", key: "some_file.txt" 85 | b\put_file_string "yeah", key: "something/with/path.cpp" 86 | b\put_file "hi.lua", key: "cool/thing.lua" 87 | 88 | moon.p b\list! 89 | 90 | b\delete_file "some_file.txt" 91 | b\delete_file "cool/does_not_exist.txt" 92 | 93 | moon.p b\list! 94 | 95 | print b\file_url "cool/does_not_exist.txt" 96 | 97 | { :MockStorage } 98 | -------------------------------------------------------------------------------- /cloud_storage/oauth.lua: -------------------------------------------------------------------------------- 1 | local url = require("socket.url") 2 | local mime = require("mime") 3 | local json = require("cjson") 4 | local pkey = require("openssl.pkey") 5 | local digest = require("openssl.digest") 6 | local h = require("cloud_storage.http") 7 | local param 8 | param = function(tbl) 9 | local tuples 10 | do 11 | local _accum_0 = { } 12 | local _len_0 = 1 13 | for k, v in pairs(tbl) do 14 | _accum_0[_len_0] = tostring(url.escape(k)) .. "=" .. tostring(url.escape(v)) 15 | _len_0 = _len_0 + 1 16 | end 17 | tuples = _accum_0 18 | end 19 | return table.concat(tuples, "&") 20 | end 21 | local OAuth 22 | do 23 | local _class_0 24 | local _base_0 = { 25 | auth_url = "https://accounts.google.com/o/oauth2/token", 26 | header = '{"alg":"RS256","typ":"JWT"}', 27 | digest_type = "sha256WithRSAEncryption", 28 | scope = { 29 | read_only = "https://www.googleapis.com/auth/devstorage.read_only", 30 | read_write = "https://www.googleapis.com/auth/devstorage.read_write", 31 | full_control = "https://www.googleapis.com/auth/devstorage.full_control" 32 | }, 33 | get_access_token = function(self) 34 | if not self.access_token or os.time() >= self.expires_at then 35 | self:refresh_access_token() 36 | end 37 | return self.access_token 38 | end, 39 | refresh_access_token = function(self) 40 | local http = h.get() 41 | local time = os.time() 42 | local jwt = self:_make_jwt(self.client_email, self.private_key) 43 | local req_params = param({ 44 | grant_type = "urn:ietf:params:oauth:grant-type:jwt-bearer", 45 | assertion = jwt 46 | }) 47 | local res = assert(http.request(self.auth_url, req_params)) 48 | res = json.decode(res) 49 | if res.error then 50 | error("Failed auth: " .. tostring(res.error)) 51 | end 52 | self.expires_at = time + res.expires_in 53 | self.access_token = res.access_token 54 | return self.access_token 55 | end, 56 | sign_string = function(self, string) 57 | local d = assert(digest.new(self.digest_type)) 58 | local key = self:_private_key() 59 | d:update(string) 60 | return (mime.b64(assert(key:sign(d)))) 61 | end, 62 | _load_private_key = function(self, str) 63 | do 64 | local key = assert(pkey.new(str)) 65 | self._private_key = function() 66 | return key 67 | end 68 | return key 69 | end 70 | end, 71 | _private_key = function(self) 72 | return self:_load_private_key(assert(assert(io.open(self.private_key_file)):read("*a"))) 73 | end, 74 | _make_jwt = function(self, client_email, private_key) 75 | local hr = 60 * 60 76 | local claims = json.encode({ 77 | iss = client_email, 78 | aud = self.auth_url, 79 | scope = self.scope.full_control, 80 | iat = os.time(), 81 | exp = os.time() + hr 82 | }) 83 | local sig_input = mime.b64(self.header) .. "." .. mime.b64(claims) 84 | local signature = self:sign_string(sig_input) 85 | return sig_input .. "." .. signature 86 | end 87 | } 88 | _base_0.__index = _base_0 89 | _class_0 = setmetatable({ 90 | __init = function(self, client_email, private_key_file) 91 | self.client_email, self.private_key_file = client_email, private_key_file 92 | end, 93 | __base = _base_0, 94 | __name = "OAuth" 95 | }, { 96 | __index = _base_0, 97 | __call = function(cls, ...) 98 | local _self_0 = setmetatable({}, _base_0) 99 | cls.__init(_self_0, ...) 100 | return _self_0 101 | end 102 | }) 103 | _base_0.__class = _class_0 104 | OAuth = _class_0 105 | end 106 | return { 107 | OAuth = OAuth 108 | } 109 | -------------------------------------------------------------------------------- /cloud_storage/multipart.lua: -------------------------------------------------------------------------------- 1 | local mimetypes = require("mimetypes") 2 | local url = require("socket.url") 3 | local insert, concat 4 | do 5 | local _obj_0 = table 6 | insert, concat = _obj_0.insert, _obj_0.concat 7 | end 8 | math.randomseed(os.time()) 9 | local type 10 | type = require("moon").type 11 | local File 12 | do 13 | local _class_0 14 | local _base_0 = { 15 | mime = function(self) 16 | return mimetypes.guess(self.fname) 17 | end, 18 | content = function(self) 19 | do 20 | local file = io.open(self.fname) 21 | if file then 22 | do 23 | local _with_0 = file:read("*a") 24 | file:close() 25 | return _with_0 26 | end 27 | end 28 | end 29 | end 30 | } 31 | _base_0.__index = _base_0 32 | _class_0 = setmetatable({ 33 | __init = function(self, fname) 34 | self.fname = fname 35 | end, 36 | __base = _base_0, 37 | __name = "File" 38 | }, { 39 | __index = _base_0, 40 | __call = function(cls, ...) 41 | local _self_0 = setmetatable({}, _base_0) 42 | cls.__init(_self_0, ...) 43 | return _self_0 44 | end 45 | }) 46 | _base_0.__class = _class_0 47 | File = _class_0 48 | end 49 | local rand_string 50 | rand_string = function(len) 51 | local shuffled 52 | do 53 | local _accum_0 = { } 54 | local _len_0 = 1 55 | for i = 1, len do 56 | local r = math.random(97, 122) 57 | if math.random() >= 0.5 then 58 | r = r - 32 59 | end 60 | local _value_0 = r 61 | _accum_0[_len_0] = _value_0 62 | _len_0 = _len_0 + 1 63 | end 64 | shuffled = _accum_0 65 | end 66 | return string.char(unpack(shuffled)) 67 | end 68 | local encode 69 | encode = function(params) 70 | local chunks 71 | do 72 | local _accum_0 = { } 73 | local _len_0 = 1 74 | for _index_0 = 1, #params do 75 | local tuple = params[_index_0] 76 | local k, v = unpack(tuple) 77 | k = url.escape(k) 78 | local buffer = { 79 | 'Content-Disposition: form-data; name="' .. k .. '"' 80 | } 81 | local content 82 | if type(v) == File then 83 | local _update_0 = 1 84 | buffer[_update_0] = buffer[_update_0] .. ('; filename="' .. v.fname .. '"') 85 | insert(buffer, "Content-type: " .. tostring(v:mime())) 86 | content = v:content() 87 | else 88 | content = v 89 | end 90 | insert(buffer, "") 91 | insert(buffer, content) 92 | local _value_0 = concat(buffer, "\r\n") 93 | _accum_0[_len_0] = _value_0 94 | _len_0 = _len_0 + 1 95 | end 96 | chunks = _accum_0 97 | end 98 | local boundary 99 | while true do 100 | boundary = "Boundary" .. tostring(rand_string(16)) 101 | for _index_0 = 1, #chunks do 102 | local _continue_0 = false 103 | repeat 104 | local c = chunks[_index_0] 105 | if c:find(boundary) then 106 | _continue_0 = true 107 | break 108 | end 109 | _continue_0 = true 110 | until true 111 | if not _continue_0 then 112 | break 113 | end 114 | end 115 | do 116 | break 117 | end 118 | end 119 | local inner = concat({ 120 | "\r\n", 121 | "--", 122 | boundary, 123 | "\r\n" 124 | }) 125 | return (concat({ 126 | "--", 127 | boundary, 128 | "\r\n", 129 | concat(chunks, inner), 130 | "\r\n", 131 | "--", 132 | boundary, 133 | "--", 134 | "\r\n" 135 | })), boundary 136 | end 137 | local encode_tbl 138 | encode_tbl = function(params) 139 | return encode((function() 140 | local _accum_0 = { } 141 | local _len_0 = 1 142 | for k, v in pairs(params) do 143 | _accum_0[_len_0] = { 144 | k, 145 | v 146 | } 147 | _len_0 = _len_0 + 1 148 | end 149 | return _accum_0 150 | end)()) 151 | end 152 | return { 153 | encode = encode, 154 | encode_tbl = encode_tbl, 155 | File = File 156 | } 157 | -------------------------------------------------------------------------------- /cloud_storage/mock.lua: -------------------------------------------------------------------------------- 1 | local Bucket 2 | Bucket = require("cloud_storage.google").Bucket 3 | local shell_escape 4 | shell_escape = function(str) 5 | return str:gsub("'", "''") 6 | end 7 | local execute 8 | execute = function(cmd) 9 | local proc = io.popen(cmd) 10 | local out = proc:read("*a") 11 | proc:close() 12 | return out:match("^(.-)%s*$") 13 | end 14 | local MockStorage 15 | do 16 | local _class_0 17 | local _base_0 = { 18 | bucket = function(self, bucket) 19 | return Bucket(bucket, self) 20 | end, 21 | _full_path = function(self, bucket, key) 22 | local dir 23 | if self.dir_name == "." then 24 | dir = "" 25 | else 26 | dir = self.dir_name .. "/" 27 | end 28 | return tostring(dir) .. tostring(bucket) .. "/" .. tostring(key) 29 | end, 30 | file_url = function(self, bucket, key) 31 | local prefix 32 | if self.url_prefix == "" then 33 | prefix = "" 34 | else 35 | prefix = self.url_prefix .. "/" 36 | end 37 | return prefix .. self:_full_path(bucket, key) 38 | end, 39 | get_service = function(self) 40 | return error("Not implemented") 41 | end, 42 | get_bucket = function(self, bucket) 43 | local path = tostring(self.dir_name) .. "/" .. tostring(bucket) 44 | execute("mkdir -p '" .. tostring(shell_escape(path)) .. "'") 45 | local escaped_path = "$(echo '" .. tostring(shell_escape(path)) .. "' | sed -e 's/[\\/&]/\\\\&/g')" 46 | local cmd = 'find "' .. shell_escape(path) .. '" -type f | sed -e "s/^' .. escaped_path .. '//"' 47 | local files = execute(cmd) 48 | return (function() 49 | local _accum_0 = { } 50 | local _len_0 = 1 51 | for file in files:gmatch("[^\n]+") do 52 | _accum_0[_len_0] = { 53 | key = file:match("/?(.*)") 54 | } 55 | _len_0 = _len_0 + 1 56 | end 57 | return _accum_0 58 | end)() 59 | end, 60 | put_file_string = function(self, bucket, key, data, options) 61 | if options == nil then 62 | options = { } 63 | end 64 | assert(not options.key, "key is not an option, but an argument") 65 | if type(data) == "table" then 66 | error("put_file_string interface has changed: key is now the second argument") 67 | end 68 | assert(key, "missing key") 69 | assert(type(data) == "string", "expected string for data") 70 | local path = self:_full_path(bucket, key) 71 | local dir = execute("dirname '" .. tostring(shell_escape(path)) .. "'") 72 | execute("mkdir -p '" .. tostring(shell_escape(dir)) .. "'") 73 | do 74 | local _with_0 = io.open(path, "w") 75 | _with_0:write(data) 76 | _with_0:close() 77 | end 78 | return 200 79 | end, 80 | put_file = function(self, bucket, fname, options) 81 | if options == nil then 82 | options = { } 83 | end 84 | local data 85 | do 86 | local f = io.open(fname) 87 | if f then 88 | do 89 | local _with_0 = f:read("*a") 90 | f:close() 91 | data = _with_0 92 | end 93 | else 94 | data = error("Failed to read file: " .. tostring(fname)) 95 | end 96 | end 97 | return self:put_file_string(bucket, data, options) 98 | end, 99 | delete_file = function(self, bucket, key) 100 | local path = self:_full_path(bucket, key) 101 | os.execute("[ -a '" .. tostring(shell_escape(path)) .. "' ] && rm '" .. tostring(shell_escape(path)) .. "'") 102 | return 200 103 | end, 104 | get_file = function(self, bucket, key) 105 | return error("not implemented") 106 | end, 107 | head_file = function(self, bucket, key) 108 | return error("Not implemented") 109 | end 110 | } 111 | _base_0.__index = _base_0 112 | _class_0 = setmetatable({ 113 | __init = function(self, dir_name, url_prefix) 114 | if dir_name == nil then 115 | dir_name = "." 116 | end 117 | if url_prefix == nil then 118 | url_prefix = "" 119 | end 120 | self.dir_name, self.url_prefix = dir_name, url_prefix 121 | end, 122 | __base = _base_0, 123 | __name = "MockStorage" 124 | }, { 125 | __index = _base_0, 126 | __call = function(cls, ...) 127 | local _self_0 = setmetatable({}, _base_0) 128 | cls.__init(_self_0, ...) 129 | return _self_0 130 | end 131 | }) 132 | _base_0.__class = _class_0 133 | MockStorage = _class_0 134 | end 135 | if ... == "test" then 136 | local moon = require("moon") 137 | local s = MockStorage("test_storage", "static") 138 | print(s:_full_path("dad_bucket", "eat/my/sucks")) 139 | print(MockStorage():_full_path("nobucket", "hello.world")) 140 | print() 141 | local b = s:bucket("my_bucket") 142 | b:put_file_string("this is a file", { 143 | key = "some_file.txt" 144 | }) 145 | b:put_file_string("yeah", { 146 | key = "something/with/path.cpp" 147 | }) 148 | b:put_file("hi.lua", { 149 | key = "cool/thing.lua" 150 | }) 151 | moon.p(b:list()) 152 | b:delete_file("some_file.txt") 153 | b:delete_file("cool/does_not_exist.txt") 154 | moon.p(b:list()) 155 | print(b:file_url("cool/does_not_exist.txt")) 156 | end 157 | return { 158 | MockStorage = MockStorage 159 | } 160 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # Test Coverage TODO 2 | 3 | This document tracks missing test coverage areas in the cloud_storage library. Tests should be added to `spec/cloud_storage_spec.moon` using the Busted framework. 4 | 5 | ## Priority: CRITICAL (Core Authentication & Infrastructure) 6 | 7 | ### OAuth Class Methods (5/5 tested) ✅ 8 | - [x] `OAuth:get_access_token()` - Token retrieval with caching 9 | - [x] `OAuth:refresh_access_token()` - Token refresh from Google OAuth 10 | - [x] `OAuth:sign_string(string)` - Cryptographic string signing 11 | - [x] `OAuth:_load_private_key(str)` - Private key loading from string 12 | - [x] `OAuth:_private_key()` - Private key loading from file 13 | - [x] Error handling for invalid keys/authentication failures 14 | 15 | ### LOMFormatter Class Methods (0/4 tested) 16 | - [ ] `LOMFormatter:format(res, code, headers)` - XML response formatting 17 | - [ ] `LOMFormatter:ListAllMyBucketsResult(res)` - Bucket list XML parsing 18 | - [ ] `LOMFormatter:ListBucketResult(res)` - File list XML parsing 19 | - [ ] `LOMFormatter:Error(res)` - Error response XML parsing 20 | 21 | ## Priority: HIGH (Missing Core Storage Operations) 22 | 23 | ### CloudStorage Class Methods (14/15 well-tested) ✅ 24 | - [x] `CloudStorage:get_service()` - Service endpoint functionality 25 | - [x] `CloudStorage:head_file(bucket, key)` - File metadata retrieval 26 | - [x] `CloudStorage:put_file(bucket, fname, options)` - File upload from filesystem 27 | - [x] `CloudStorage:put_file_acl(bucket, key, acl)` - ACL management 28 | - [x] `CloudStorage:compose(bucket, key, source_keys, options)` - File composition 29 | - [x] `CloudStorage:start_resumable_upload(bucket, key, options)` - Resumable uploads 30 | - [x] `CloudStorage:encode_and_sign_policy(expiration, conditions)` - Policy encoding 31 | - [x] Comprehensive error handling for all methods with validation 32 | - [ ] Additional edge cases (malformed inputs, network failures, etc.) 33 | 34 | ## Priority: MEDIUM (Alternative Implementations) 35 | 36 | ### MockStorage Class Methods (0/11 tested) 37 | - [ ] `MockStorage:new(dir_name, url_prefix)` - Constructor 38 | - [ ] `MockStorage:bucket(bucket)` - Bucket wrapper creation 39 | - [ ] `MockStorage:_full_path(bucket, key)` - Path construction 40 | - [ ] `MockStorage:file_url(bucket, key)` - URL generation 41 | - [ ] `MockStorage:get_service()` - Service endpoint (should error) 42 | - [ ] `MockStorage:get_bucket(bucket)` - File listing via filesystem 43 | - [ ] `MockStorage:put_file_string(bucket, key, data, options)` - String upload 44 | - [ ] `MockStorage:put_file(bucket, fname, options)` - File upload 45 | - [ ] `MockStorage:delete_file(bucket, key)` - File deletion 46 | - [ ] `MockStorage:get_file(bucket, key)` - File retrieval (should error) 47 | - [ ] `MockStorage:head_file(bucket, key)` - File metadata (should error) 48 | 49 | ### Multipart Module Functions (0/5 tested) 50 | - [ ] `encode(params)` - Multipart form encoding 51 | - [ ] `encode_tbl(params)` - Table-based multipart encoding 52 | - [ ] `File:new(fname)` - File object constructor 53 | - [ ] `File:mime()` - MIME type detection 54 | - [ ] `File:content()` - File content reading 55 | 56 | ### Bucket Class Methods (0/2+ tested) 57 | - [ ] `Bucket:new(bucket_name, storage)` - Constructor 58 | - [ ] Method forwarding functionality (list, delete_file, get_file, etc.) 59 | - [ ] Integration with both CloudStorage and MockStorage backends 60 | 61 | ## Priority: LOW (Future/Incomplete Features) 62 | 63 | ### R2 Module Methods (0/2 tested) 64 | - [ ] `CloudflareR2:base_url(account_id)` - URL construction 65 | - [ ] `CloudflareR2:list_buckets()` - Bucket listing (incomplete implementation) 66 | 67 | ### Utility Functions 68 | - [ ] `url_encode_key()` - Additional edge cases beyond current tests 69 | 70 | ## Integration Test Areas (0/4 tested) 71 | 72 | - [ ] **Authentication Workflows** - End-to-end OAuth flow with real/mock responses 73 | - [ ] **File Upload/Download Workflows** - Complete file lifecycle testing 74 | - [ ] **Cross-Storage Compatibility** - Same operations work on CloudStorage and MockStorage 75 | - [ ] **Error Propagation** - Errors bubble up correctly through Bucket -> Storage layers 76 | 77 | ## Testing Guidelines 78 | 79 | ### Test Structure 80 | - Follow existing patterns in `spec/cloud_storage_spec.moon` 81 | - Use `describe` blocks to group related functionality 82 | - Use `before_each`/`after_each` for setup/teardown 83 | - Mock HTTP requests for CloudStorage tests 84 | - Use filesystem operations for MockStorage tests 85 | 86 | ### Key Testing Principles 87 | - Test both success and failure cases 88 | - Validate input parameter checking (nil, empty string, wrong type) 89 | - Verify HTTP request structure for CloudStorage methods 90 | - Check file system operations for MockStorage methods 91 | - Test authentication token caching and refresh logic 92 | - Validate XML parsing with various response formats 93 | 94 | ### Dependencies 95 | - Some tests require test key files (spec/test_key.pem, spec/test_key.json) 96 | - HTTP mocking requires the existing test infrastructure 97 | - MockStorage tests need temporary directory handling 98 | - LOMFormatter tests need sample XML responses 99 | 100 | ## Progress Tracking 101 | 102 | **Total Methods Identified**: ~35 103 | **Currently Well-Tested**: ~10 104 | **Coverage Goal**: 90%+ of public methods 105 | 106 | Update this file as tests are added and remove completed items. -------------------------------------------------------------------------------- /tags: -------------------------------------------------------------------------------- 1 | !_TAG_FILE_FORMAT 2 /extended format/ 2 | !_TAG_FILE_SORTED 1 /0=unsorted, 1=sorted, 2=foldcase/ 3 | !_TAG_PROGRAM_AUTHOR leaf corcoran /leafot@gmail.com/ 4 | !_TAG_PROGRAM_NAME MoonTags // 5 | !_TAG_PROGRAM_URL https://github.com/leafo/moonscript /GitHub repository/ 6 | !_TAG_PROGRAM_VERSION 0.0.1 // 7 | 8 | Bucket cloud_storage/google.moon /^class Bucket$/;" c language:moon 9 | CloudStorage cloud_storage/google.moon /^class CloudStorage$/;" c language:moon 10 | File cloud_storage/multipart.moon /^class File$/;" c language:moon 11 | LOMFormatter cloud_storage/google.moon /^class LOMFormatter$/;" c language:moon 12 | MockStorage cloud_storage/mock.moon /^class MockStorage$/;" c language:moon 13 | OAuth cloud_storage/oauth.moon /^class OAuth$/;" c language:moon 14 | _full_path cloud_storage/mock.moon /^ _full_path: (bucket, key) =>/;" f language:moon class:MockStorage 15 | _headers cloud_storage/google.moon /^ _headers: =>/;" f language:moon class:CloudStorage 16 | _load_private_key cloud_storage/oauth.moon /^ _load_private_key: (str) =>/;" f language:moon class:OAuth 17 | _make_jwt cloud_storage/oauth.moon /^ _make_jwt: (client_email, private_key) =>/;" f language:moon class:OAuth 18 | _private_key cloud_storage/oauth.moon /^ _private_key: =>/;" f language:moon class:OAuth 19 | _request cloud_storage/google.moon /^ _request: (method="GET", path, data, headers) =>/;" f language:moon class:CloudStorage 20 | bucket cloud_storage/google.moon /^ bucket: (bucket) =>/;" f language:moon class:CloudStorage 21 | bucket cloud_storage/mock.moon /^ bucket: (bucket) =>/;" f language:moon class:MockStorage 22 | bucket_url cloud_storage/google.moon /^ bucket_url: (bucket, opts={}) =>/;" f language:moon class:CloudStorage 23 | canonicalize_headers cloud_storage/google.moon /^ canonicalize_headers: (headers) =>/;" f language:moon class:CloudStorage 24 | compose cloud_storage/google.moon /^ compose: (bucket, key, source_keys, options={}) =>/;" f language:moon class:CloudStorage 25 | content cloud_storage/multipart.moon /^ content: =>/;" f language:moon class:File 26 | copy_file cloud_storage/google.moon /^ copy_file: (source_bucket, source_key, dest_bucket, dest_key, options={}) =>/;" f language:moon class:CloudStorage 27 | delete_file cloud_storage/google.moon /^ delete_file: (bucket, key) =>/;" f language:moon class:CloudStorage 28 | delete_file cloud_storage/mock.moon /^ delete_file: (bucket, key) =>/;" f language:moon class:MockStorage 29 | encode cloud_storage/multipart.moon /^encode = (params) ->/;" f language:moon 30 | encode_and_sign_policy cloud_storage/google.moon /^ encode_and_sign_policy: (expiration, conditions) =>/;" f language:moon class:CloudStorage 31 | encode_tbl cloud_storage/multipart.moon /^encode_tbl = (params) ->/;" f language:moon 32 | file_url cloud_storage/google.moon /^ file_url: (bucket, key, opts) =>/;" f language:moon class:CloudStorage 33 | file_url cloud_storage/mock.moon /^ file_url: (bucket, key) =>/;" f language:moon class:MockStorage 34 | format cloud_storage/google.moon /^ format: (res, code, headers) =>/;" f language:moon class:LOMFormatter 35 | from_json_key_file cloud_storage/google.moon /^ @from_json_key_file: (file) =>/;" f language:moon class:CloudStorage 36 | get cloud_storage/http.moon /^get = ->/;" f language:moon 37 | get_access_token cloud_storage/oauth.moon /^ get_access_token: =>/;" f language:moon class:OAuth 38 | get_bucket cloud_storage/google.moon /^ get_bucket: (bucket) =>/;" f language:moon class:CloudStorage 39 | get_bucket cloud_storage/mock.moon /^ get_bucket: (bucket) =>/;" f language:moon class:MockStorage 40 | get_file cloud_storage/google.moon /^ get_file: (bucket, key, opts) =>/;" f language:moon class:CloudStorage 41 | get_file cloud_storage/mock.moon /^ get_file: (bucket, key) =>/;" f language:moon class:MockStorage 42 | get_service cloud_storage/google.moon /^ get_service: =>/;" f language:moon class:CloudStorage 43 | get_service cloud_storage/mock.moon /^ get_service: =>/;" f language:moon class:MockStorage 44 | head_file cloud_storage/google.moon /^ head_file: (bucket, key) =>/;" f language:moon class:CloudStorage 45 | head_file cloud_storage/mock.moon /^ head_file: (bucket, key) =>/;" f language:moon class:MockStorage 46 | mime cloud_storage/multipart.moon /^ mime: =>/;" f language:moon class:File 47 | new cloud_storage/google.moon /^ new: (@bucket_name, @storage) =>/;" f language:moon class:Bucket 48 | new cloud_storage/google.moon /^ new: (@oauth, @project_id) =>/;" f language:moon class:CloudStorage 49 | new cloud_storage/google.moon /^ new: =>/;" f language:moon class:LOMFormatter 50 | new cloud_storage/mock.moon /^ new: (@dir_name="\.", @url_prefix="") =>/;" f language:moon class:MockStorage 51 | new cloud_storage/multipart.moon /^ new: (@fname) =>/;" f language:moon class:File 52 | new cloud_storage/oauth.moon /^ new: (@client_email, @private_key_file) =>/;" f language:moon class:OAuth 53 | put_file cloud_storage/google.moon /^ put_file: (bucket, fname, options={}) =>/;" f language:moon class:CloudStorage 54 | put_file cloud_storage/mock.moon /^ put_file: (bucket, fname, options={}) =>/;" f language:moon class:MockStorage 55 | put_file_acl cloud_storage/google.moon /^ put_file_acl: (bucket, key, acl) =>/;" f language:moon class:CloudStorage 56 | put_file_string cloud_storage/google.moon /^ put_file_string: (bucket, key, data, options={}) =>/;" f language:moon class:CloudStorage 57 | put_file_string cloud_storage/mock.moon /^ put_file_string: (bucket, key, data, options={}) =>/;" f language:moon class:MockStorage 58 | refresh_access_token cloud_storage/oauth.moon /^ refresh_access_token: =>/;" f language:moon class:OAuth 59 | set cloud_storage/http.moon /^set = (http) ->/;" f language:moon 60 | sign_string cloud_storage/oauth.moon /^ sign_string: (string) =>/;" f language:moon class:OAuth 61 | signed_url cloud_storage/google.moon /^ signed_url: (bucket, key, expiration, opts={}) =>/;" f language:moon class:CloudStorage 62 | start_resumable_upload cloud_storage/google.moon /^ start_resumable_upload: (bucket, key, options={}) =>/;" f language:moon class:CloudStorage 63 | upload_url cloud_storage/google.moon /^ upload_url: (bucket, key, opts={}) =>/;" f language:moon class:CloudStorage 64 | url_encode_key cloud_storage/google.moon /^url_encode_key = (key) ->/;" f language:moon 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `cloud_storage` 2 | 3 | ![test](https://github.com/leafo/cloud_storage/workflows/test/badge.svg) 4 | 5 | A library for connecting to [Google Cloud Storage](https://cloud.google.com/products/cloud-storage) through Lua. 6 | 7 | ## Tutorial 8 | 9 | You can learn more about authenticating with Google Cloud Storage here: 10 | . 11 | 12 | Here's a quick guide to get you started: 13 | 14 | The easiest way use this library is to create a service account for your 15 | project. You'll need to generate a private key and store it alongside your 16 | configuration. 17 | 18 | Go to the cloud console: . Enable Cloud 19 | Storage if you haven't done so already. You may also need to enter billing 20 | information. 21 | 22 | Navigate to **IAM & admin** » **Service accounts**, located on the sidebar. 23 | Click **Create service account**. 24 | 25 | ![Create service account](http://leafo.net/shotsnb/2018-03-31_20-54-45.png) 26 | 27 | Select **Furnish a new private key** and select **JSON** as the type. After 28 | creating the service account your browser will download a `.json` file. Store 29 | that securely, since it grants access to your project. 30 | 31 | The `.json` is used to create a new API client with this library: 32 | 33 | ```lua 34 | local google = require "cloud_storage.google" 35 | local storage = google.CloudStorage:from_json_key_file("path/to/my-key.json") 36 | 37 | -- you can now use any of the methods on the storage object 38 | local files = assert(storage:get_bucket("my_bucket")) 39 | ``` 40 | 41 |
42 | 43 | 44 | 45 | ## Using a p12/pem secret key 46 | 47 | 48 | 49 | Use these directions only if you have a key created with the `P12` type. 50 | 51 | The private key downloaded from the Cloud Console is a `.p12` file. The key 52 | uses a hard coded password `notasecret`. 53 | 54 | In order to use it, it must be converted to a `.pem` file. Run the following 55 | command (replacing `key.p12` and `key.pem` with the input filename and desired 56 | output filename). Enter `notasecret` for the password. 57 | 58 | ```bash 59 | openssl pkcs12 -in key.p12 -out key.pem -nodes -clcerts 60 | ``` 61 | 62 | One more piece of information is needed: the service account email address. 63 | You'll find it labeled **Service account ID** on the service account list. It 64 | might look something like `cloud-storage@my-project.iam.gserviceaccount.com`. 65 | 66 | You can now connect to the API. Create a new client like this: 67 | 68 | ```lua 69 | local oauth = require "cloud_storage.oauth" 70 | local google = require "cloud_storage.google" 71 | 72 | -- replace with your service account ID 73 | local o = oauth.OAuth("cloud-storage@my-project.iam.gserviceaccount.com", "path/to/key.pem") 74 | 75 | -- use your id as the second argument, everything before the @ in your service account ID 76 | local storage = google.CloudStorage(o, "cloud-storage") 77 | 78 | local files = assert(storage:get_bucket("my_bucket")) 79 | ``` 80 | 81 |
82 | 83 | ## HTTP Client Customization 84 | 85 | By default, this library uses `socket.http` for making HTTP requests. You can provide a custom HTTP client that supports the LuaSocket interface: 86 | 87 | ```lua 88 | local http = require("cloud_storage.http") 89 | 90 | -- Replace with your custom HTTP client 91 | -- Must implement a request() function compatible with LuaSocket 92 | http.set(require("lapis.http")) 93 | 94 | -- Now all future requests will use the `request` function in your http client... 95 | ``` 96 | 97 | This allows for custom timeout handling, proxy configuration, or mock clients for testing. 98 | 99 | ## Reference 100 | 101 | ### Module `cloud_storage.oauth` 102 | 103 | This module contains the OAuth implementation used by the storage API. 104 | 105 | ```lua 106 | local oauth = require "cloud_storage.oauth" 107 | ``` 108 | 109 | #### `ouath_instance = oauth.OAuth(service_email, path_to_pem_file=nil)` 110 | 111 | Create a new OAuth object. Handles OAuth authenticated requests. 112 | 113 | It's not necessary to use this module directly if loading private key from a 114 | `.json` file. 115 | 116 | ### Module `cloud_storage.google` 117 | 118 | Communicates with the Google cloud storage API. 119 | 120 | ```lua 121 | local google = require "cloud_storage.google" 122 | ``` 123 | 124 | #### Error handling 125 | 126 | Any methods that fail to execute will return `nil`, an error message, and an 127 | object that represents the error. Successful responses will return a Lua table 128 | containing the details of the operation. 129 | 130 | #### `storage = google.CloudStorage(ouath_instance, project_id)` 131 | 132 | Create a new instance of a storage object from an OAuth instance. Note that the 133 | `from_json_key_file` interface is a simpler way of constructing the storage 134 | interface. 135 | 136 | ```lua 137 | local oauth = require "cloud_storage.oauth" 138 | local google = require "cloud_storage.google" 139 | 140 | local o = oauth.OAuth("me@my-project.iam.gserviceaccount.com", "key.pem") 141 | 142 | local storage = google.CloudStorage(o, "111111111111") 143 | ``` 144 | 145 | #### `storage = google.CloudStorage:from_json_key_file(json_key_file_name)` 146 | 147 | Create a new instance of a storage object from a `.json` private key file 148 | generated by the Google Cloud Console. 149 | 150 | ```lua 151 | local google = require "cloud_storage.google" 152 | local storage = google.CloudStorage:from_json_key_file("my-storage-auth.json") 153 | ``` 154 | 155 | #### `storage:get_service()` 156 | 157 | 158 | 159 | #### `storage:get_bucket(bucket)` 160 | 161 | 162 | 163 | #### `storage:get_file(bucket, key)` 164 | 165 | 166 | 167 | #### `storage:delete_file(bucket, key)` 168 | 169 | 170 | 171 | #### `storage:head_file(bucket, key)` 172 | 173 | 174 | 175 | #### `storage:copy_file(source_bucket, source_key, dest_bucket, dest_key, options={})` 176 | 177 | 178 | 179 | Options: 180 | 181 | * `acl`: value to use for `x-goog-acl`, defaults to `public-read` 182 | * `headers`: table of additional headers to include in request 183 | 184 | #### `storage:compose(bucket, key, source_keys, options={})` 185 | 186 | 187 | 188 | Composes a new file from multiple source files in the same bucket by 189 | concatenating them. 190 | 191 | `source_keys` is an array of keys as strings or an array of key declarations as 192 | tables. 193 | 194 | The table format for a source key is structured like this: 195 | 196 | ```lua 197 | { 198 | name = "file.png", 199 | -- optional fields: 200 | generation = "1361471441094000", 201 | if_generation_match = "1361471441094000", 202 | } 203 | ``` 204 | 205 | Options: 206 | 207 | * `acl`: value to use for `x-goog-acl`, defaults to `public-read` 208 | * `mimetype`: sets `Content-type` header for the composed file 209 | * `headers`: table of additional headers to include in request 210 | 211 | #### `storage:put_file_string(bucket, key, data, opts={})` 212 | 213 | > **Note:** this API previously had `key` as an option but it was moved to 214 | > second argument 215 | 216 | Uploads the string `data` to the bucket at the specified key. 217 | 218 | Options include: 219 | 220 | * `mimetype`: sets `Content-type` header for the file, defaults to not being set 221 | * `acl`: sets the `x-goog-acl` header for file, defaults to `public-read` 222 | * `headers`: an optional array table of any additional headers to send 223 | 224 | ```lua 225 | storage:put_file_string("my_bucket", "message.txt", "hello world!", { 226 | mimetype = "text/plain", 227 | acl = "private" 228 | }) 229 | ``` 230 | 231 | #### `storage:put_file(bucket, fname, opts={})` 232 | 233 | Reads `fname` from disk and uploads it. The key of the file will be the name of 234 | the file unless `opts.key` is provided. The mimetype of the file is guessed 235 | based on the extension unless `opts.mimetype` is provided. 236 | 237 | ```lua 238 | storage:put_file("my_bucket", "source.lua", { 239 | mimetype = "text/lua" 240 | }) 241 | ``` 242 | 243 | All the same options from `put_file_string` are available for this method. 244 | 245 | > **Note:** This method is currently inefficient . The whole file is loaded 246 | > into memory and sent at once in the request. An alternative implementation 247 | > will be added in the future. Open a ticket if you need it. 248 | 249 | #### `storage:start_resumable_upload(bucket, key, options={})` 250 | 251 | 252 | 253 | Options: 254 | 255 | * `acl`: value to use for `x-goog-acl`, defaults to `public-read` 256 | * `mimetype`: sets `Content-type` header for the composed file 257 | * `headers`: table of additional headers to include in request 258 | 259 | #### `storage:bucket_url(bucket, opts={})` 260 | 261 | Returns full URL for a bucket 262 | 263 | Options: 264 | 265 | * `subdomain`: use the `{bucket}.domain` format instead of default `domain/{bucket}` format 266 | * `scheme`: protocol of returned URL, default `https` 267 | 268 | #### `storage:bucket_url(bucket, opts={})` 269 | 270 | Returns full URL for a key on a bucket. Supports the same options as `storage:bucket_url` method. 271 | 272 | ```lua 273 | storage:file_url("my-bucket", "pics/leafo.png") --> "https://commondatastorage.googleapis.com/my-bucket/pics/leafo.png" 274 | storage:file_url("my-bucket", "pics/leafo.png", { 275 | scheme = "http", 276 | subdomain = true, 277 | }) --> "http://my-bucket.commondatastorage.googleapis.com/pics/leafo.png" 278 | ``` 279 | 280 | Options: 281 | 282 | * `subdomain`: use the `{bucket}.domain` format instead of default `domain/{bucket}` format 283 | * `scheme`: protocol of returned URL, default `https` 284 | 285 | #### `storage:signed_url(bucket, key, expiration, opts={})` 286 | 287 | Creates a temporarily URL for downloading an object regardless of it's ACL. 288 | `expiration` is a Unix timestamp in the future, like one generated from 289 | `os.time()`. 290 | 291 | ```lua 292 | print(storage:signed_url("my_bucket", "message.txt", os.time() + 100)) 293 | ``` 294 | 295 | Options: 296 | 297 | * `headers`: table of additional `x-goog` headers to embed into the signature 298 | * `verb`: HTTP verb to embed into the signature, deafult `GET` 299 | * `scheme`: protocol of returned URL, default `https` 300 | 301 | #### `storage:upload_url(bucket, key, opts={})` 302 | 303 | Generate a signed URL for browser-based upload. 304 | 305 | Options: 306 | 307 | * `content_disposition` 308 | * `filename` If provided, will append to content disposition as `filename=` (and set content disposition to `attachment` if none is provided) 309 | * `acl` default `project-private` 310 | * `success_action_redirect` 311 | * `expires` unix timestamp. Deafult: 1 hour from now 312 | * `size_limit` 313 | * `scheme` protocol of returned URL, default `https` 314 | 315 | 316 | #### `storage:put_file_acl(bucket, key, acl)` 317 | 318 | 319 | 320 | ## Changelog 321 | 322 | ### Future updates 323 | 324 | All future changelogs will be post on github release notes: https://github.com/leafo/cloud_storage/releases 325 | 326 | ### `1.0.0` Mar 31, 2018 327 | 328 | * **Changed** `put_file_string` now takes `key` as second argument, instead of within options table 329 | * Replace `luacrypto` with `luaossl` 330 | * Add support for `json` private keys 331 | * Better error handling for all API calls. (Failures return `nil`, error message, and error object) 332 | * Add `copy_file` and `compose` methods 333 | * Add `upload_url` method to create upload URLs 334 | * Add `start_resumable_upload` method for creating resumable uploads 335 | * Headers are canonicalized and sorted to make them consistent between calls 336 | * Fix bug where special characters in key were not being URL encoded 337 | * Fix bug where some special characters were not being encoded for signed URLs 338 | * Rewrite documentation tutorial 339 | 340 | ### `0.1.0` Sep 29, 2013 341 | 342 | * Initial release 343 | 344 | [0]: https://developers.google.com/storage/docs/accesscontrol 345 | -------------------------------------------------------------------------------- /cloud_storage/google.moon: -------------------------------------------------------------------------------- 1 | 2 | url = require "socket.url" 3 | date = require "date" 4 | ltn12 = require "ltn12" 5 | json = require "cjson" 6 | mime = require "mime" 7 | 8 | mimetypes = require "mimetypes" 9 | 10 | h = require "cloud_storage.http" 11 | 12 | import insert, concat from table 13 | 14 | url_encode_key = (key) -> 15 | (key\gsub [==[[%[%]#!%^%*%(%)"'%%]]==], (c) -> 16 | "%#{"%x"\format(c\byte!)\upper!}") 17 | 18 | extend = (t, ...) -> 19 | for other in *{...} 20 | if other != nil 21 | t[k] = v for k,v in pairs other 22 | t 23 | 24 | xml_escape = do 25 | punct = "[%^$()%.%[%]*+%-?]" 26 | escape_patt = (str) -> (str\gsub punct, (p) -> "%"..p) 27 | 28 | xml_escape_entities = { 29 | ['&']: '&' 30 | ['<']: '<' 31 | ['>']: '>' 32 | ['"']: '"' 33 | ["'"]: ''' 34 | } 35 | 36 | xml_unescape_entities = {} 37 | for key,value in pairs xml_escape_entities 38 | xml_unescape_entities[value] = key 39 | 40 | xml_escape_pattern = "[" .. concat([escape_patt char for char in pairs xml_escape_entities]) .. "]" 41 | 42 | (text) -> (text\gsub xml_escape_pattern, xml_escape_entities) 43 | 44 | class LOMFormatter 45 | find_node = (node, tag) -> 46 | for child in *node 47 | if child.tag == tag 48 | return child 49 | 50 | filter_nodes = (node, tag) -> 51 | return for child in *node 52 | continue unless child.tag == tag 53 | child 54 | 55 | node_value = (node, tag) -> 56 | child = find_node node, tag 57 | child and child[1] 58 | 59 | new: => 60 | @lom = require "lxp.lom" 61 | 62 | format: (res, code, headers) => 63 | return code, headers if res == "" 64 | return res if headers["x-goog-generation"] 65 | 66 | res = @lom.parse res 67 | return nil, "Failed to parse result #{code}" if not res 68 | 69 | if @[res.tag] 70 | @[res.tag] @, res 71 | else 72 | res, code 73 | 74 | "ListAllMyBucketsResult": (res) => 75 | buckets_node = find_node res, "Buckets" 76 | return for bucket in *buckets_node 77 | { 78 | name: node_value bucket, "Name" 79 | creation_date: node_value bucket, "CreationDate" 80 | } 81 | 82 | "ListBucketResult": (res) => 83 | return for node in *filter_nodes res, "Contents" 84 | { 85 | key: node_value node, "Key" 86 | size: tonumber node_value node, "Size" 87 | last_modified: node_value node, "LastModified" 88 | } 89 | 90 | "Error": (res) => 91 | { 92 | error: true 93 | message: node_value res, "Message" 94 | code: node_value res, "Code" 95 | details: node_value res, "Details" 96 | } 97 | 98 | class Bucket 99 | forward_methods = { 100 | "get_bucket": "list" 101 | "get_file" 102 | "delete_file" 103 | "head_file" 104 | "put_file" 105 | "put_file_string" 106 | "file_url" 107 | } 108 | 109 | new: (@bucket_name, @storage) => 110 | 111 | for k,v in pairs forward_methods 112 | name, self_name = if type(k) == "number" 113 | v,v 114 | else 115 | k,v 116 | 117 | @__base[self_name] = (...) => 118 | @storage[name] @storage, @bucket_name, ... 119 | 120 | class CloudStorage 121 | url_base: "commondatastorage.googleapis.com" 122 | api_base: "storage.googleapis.com" 123 | 124 | @from_json_key_file: (file) => 125 | file_contents = assert assert(io.open(file))\read "*a" 126 | json = require("cjson") 127 | obj = assert json.decode file_contents 128 | import OAuth from require "cloud_storage.oauth" 129 | 130 | oauth = OAuth obj.client_email 131 | oauth\_load_private_key obj.private_key 132 | 133 | CloudStorage oauth, obj.client_id 134 | 135 | new: (@oauth, @project_id) => 136 | @formatter = LOMFormatter! 137 | 138 | _headers: => 139 | { 140 | "x-goog-api-version": 2 141 | "x-goog-project-id": @project_id 142 | "Authorization": "OAuth #{@oauth\get_access_token!}" 143 | "Date": date!\fmt "${http}" 144 | } 145 | 146 | _request: (method="GET", path, data, headers) => 147 | http = h.get! 148 | 149 | out = {} 150 | r = { 151 | url: "https://#{@api_base}#{path}" 152 | source: data and ltn12.source.string data 153 | method: method 154 | headers: extend @_headers!, headers 155 | sink: ltn12.sink.table out 156 | } 157 | 158 | _, code, res_headers = http.request r 159 | res, code = @formatter\format table.concat(out), code, res_headers 160 | 161 | if type(res) == "table" and res.error 162 | nil, "#{res.message} #{res.details}", res 163 | else 164 | res, code 165 | 166 | bucket: (bucket) => Bucket bucket, @ 167 | 168 | file_url: (bucket, key, opts) => 169 | @bucket_url(bucket, opts) .. "/#{key}" 170 | 171 | bucket_url: (bucket, opts={}) => 172 | scheme = opts.scheme or "https" 173 | if opts.subdomain 174 | "#{scheme}://#{bucket}.#{@url_base}" 175 | else 176 | "#{scheme}://#{@url_base}/#{bucket}" 177 | 178 | for m in *{"GET", "POST", "PUT", "DELETE", "HEAD"} 179 | @__base["_#{m\lower!}"] = (...) => @_request m, ... 180 | 181 | get_service: => @_get "/" 182 | get_bucket: (bucket) => @_get "/#{bucket}" 183 | 184 | get_file: (bucket, key, opts) => 185 | assert type(key) == "string" and key != "", "Invalid key (missing or empty string)" 186 | @_get "/#{bucket}/#{url.escape key}", nil, opts and opts.headers 187 | 188 | delete_file: (bucket, key) => 189 | assert type(key) == "string" and key != "", "Invalid key for deletion (missing or empty string)" 190 | @_delete "/#{bucket}/#{url.escape key}" 191 | 192 | head_file: (bucket, key) => 193 | assert type(key) == "string" and key != "", "Invalid key (missing or empty string)" 194 | @_head "/#{bucket}/#{url.escape key}" 195 | 196 | -- sets predefined acl 197 | put_file_acl: (bucket, key, acl) => 198 | assert type(key) == "string" and key != "", "Invalid key (missing or empty string)" 199 | @_put "/#{bucket}/#{url.escape key}?acl", "", { 200 | "Content-length": 0 201 | "x-goog-acl": acl 202 | } 203 | 204 | put_file_string: (bucket, key, data, options={}) => 205 | assert not options.key, "key is not an option, but an argument" 206 | if type(data) == "table" 207 | error "put_file_string interface has changed: key is now the second argument" 208 | 209 | assert type(key) == "string" and key != "", "Invalid key (missing or empty string)" 210 | assert type(data) == "string", "expected string for data" 211 | 212 | @_put "/#{bucket}/#{url.escape key}", data, extend { 213 | "Content-length": #data 214 | "Content-type": options.mimetype 215 | "x-goog-acl": options.acl or "public-read" 216 | }, options.headers 217 | 218 | put_file: (bucket, fname, options={}) => 219 | data = if f = io.open fname 220 | with f\read "*a" 221 | f\close! 222 | else 223 | error "Failed to read file: #{fname}" 224 | 225 | options.mimetype or= mimetypes.guess fname 226 | key = options.key or fname 227 | @put_file_string bucket, key, data, options 228 | 229 | copy_file: (source_bucket, source_key, dest_bucket, dest_key, options={}) => 230 | @_put "/#{dest_bucket}/#{url.escape dest_key}", "", extend { 231 | "Content-length": "0" 232 | "x-goog-copy-source": "/#{source_bucket}/#{source_key}" 233 | "x-goog-acl": options.acl or "public-read" 234 | "x-goog-metadata-directive": options.metadata_directive 235 | }, options.headers 236 | 237 | compose: (bucket, key, source_keys, options={}) => 238 | assert type(source_keys) == "table" and next(source_keys), "invalid source keys" 239 | 240 | payload_buffer = {""} 241 | for key_obj in *source_keys 242 | local name, generation, if_generation_match 243 | 244 | if type(key_obj) == "table" 245 | {:name, :generation, :if_generation_match} = key_obj 246 | else 247 | name = key_obj 248 | 249 | assert name, "missing source key name for compose" 250 | table.insert payload_buffer, "" 251 | table.insert payload_buffer, "#{xml_escape name}" 252 | 253 | if generation 254 | table.insert payload_buffer, "#{xml_escape generation}" 255 | 256 | if if_generation_match 257 | table.insert payload_buffer, "#{xml_escape if_generation_match}" 258 | 259 | table.insert payload_buffer, "" 260 | 261 | table.insert payload_buffer, "" 262 | 263 | payload = table.concat payload_buffer 264 | 265 | @_put "/#{bucket}/#{url.escape key}?compose", payload, extend { 266 | "Content-length": #payload 267 | "x-goog-acl": options.acl or "public-read" 268 | "Content-type": options.mimetype 269 | }, options.headers 270 | 271 | start_resumable_upload: (bucket, key, options={}) => 272 | assert bucket, "missing bucket" 273 | assert key, "missing key" 274 | 275 | if type(key) == "table" 276 | options = key 277 | key = assert options.key, "missing key" 278 | 279 | @_post "/#{bucket}/#{url.escape key}", "", extend { 280 | "Content-type": options.mimetype 281 | "Content-length": 0 282 | "x-goog-acl": options.acl or "public-read" 283 | "x-goog-resumable": "start" 284 | }, options.headers 285 | 286 | canonicalize_headers: (headers) => 287 | header_pairs = [{k\lower!,v} for k, v in pairs headers] 288 | -- only count custom headers (x-goog), omit secret encryption headers 289 | header_pairs = [e for e in *header_pairs when (e[1]\match("x%-goog.*") and not e[1]\match("x%-goog%-encryption%-key.*"))] 290 | 291 | table.sort header_pairs, (a, b) -> 292 | a[1] < b[1] 293 | -- replace folding whitespace with spaces 294 | values = [e[1] .. ":" .. e[2]\gsub("\r?\n", " ") for e in *header_pairs] 295 | return concat values, "\n" 296 | 297 | encode_and_sign_policy: (expiration, conditions) => 298 | if type(expiration) == "number" 299 | expiration = os.date "!%Y-%m-%dT%H:%M:%SZ", expiration 300 | 301 | -- this is done this way to ensure stable ordering for specs 302 | doc = mime.b64 [[{"expiration":]] ..json.encode(expiration).. [[,"conditions":]] .. json.encode(conditions) .. [[}]] 303 | doc, @oauth\sign_string doc 304 | 305 | -- expiration: unix timestamp in UTC 306 | signed_url: (bucket, key, expiration, opts={}) => 307 | {:headers, :verb, :scheme} = opts 308 | 309 | verb or= "GET" 310 | scheme or= "https" 311 | 312 | key = url_encode_key key 313 | 314 | path = "/#{bucket}/#{key}" 315 | expiration = tostring expiration 316 | 317 | elements = { 318 | verb 319 | "" -- md5 320 | "" -- content-type 321 | expiration 322 | } 323 | 324 | -- 'As Needed', not required 325 | if headers and next headers 326 | table.insert elements, @canonicalize_headers headers 327 | 328 | table.insert elements, "" -- trailing newline 329 | 330 | str = concat elements, "\n" 331 | str ..= path 332 | 333 | signature = @oauth\sign_string str 334 | 335 | escape = (str) -> 336 | (str\gsub "[/+]", { 337 | "+": "%2B" 338 | "/": "%2F" 339 | }) 340 | 341 | concat { 342 | scheme 343 | "://" 344 | @url_base 345 | path 346 | "?GoogleAccessId=", @oauth.client_email 347 | "&Expires=", expiration 348 | "&Signature=", escape signature 349 | } 350 | 351 | upload_url: (bucket, key, opts={}) => 352 | { 353 | :content_disposition, :filename, :acl, :success_action_redirect, 354 | :expires, :size_limit, :scheme 355 | } = opts 356 | 357 | expires or= os.time! + 60^2 358 | acl or= "project-private" 359 | 360 | if filename 361 | content_disposition or= "attachment" 362 | filename_quoted = filename\gsub '"', "\\%1" 363 | content_disposition ..= "; filename=\"#{filename_quoted}\"" 364 | 365 | policy = {} 366 | insert policy, { :acl } 367 | insert policy, { :bucket } 368 | insert policy, {"eq", "$key", key} 369 | 370 | if content_disposition 371 | insert policy, {"eq", "$Content-Disposition", content_disposition} 372 | 373 | if size_limit 374 | insert policy, {"content-length-range", 0, size_limit} 375 | 376 | if success_action_redirect 377 | insert policy, { :success_action_redirect } 378 | 379 | policy, signature = @encode_and_sign_policy expires, policy 380 | 381 | action = @bucket_url bucket, subdomain: true, :scheme 382 | 383 | params = { 384 | :acl, :policy, :signature, :key 385 | :success_action_redirect 386 | 387 | "Content-Disposition": content_disposition 388 | GoogleAccessId: @oauth.client_email 389 | } 390 | 391 | action, params 392 | 393 | 394 | { :CloudStorage, :Bucket, :url_encode_key } 395 | -------------------------------------------------------------------------------- /cloud_storage/google.lua: -------------------------------------------------------------------------------- 1 | local url = require("socket.url") 2 | local date = require("date") 3 | local ltn12 = require("ltn12") 4 | local json = require("cjson") 5 | local mime = require("mime") 6 | local mimetypes = require("mimetypes") 7 | local h = require("cloud_storage.http") 8 | local insert, concat 9 | do 10 | local _obj_0 = table 11 | insert, concat = _obj_0.insert, _obj_0.concat 12 | end 13 | local url_encode_key 14 | url_encode_key = function(key) 15 | return (key:gsub([==[[%[%]#!%^%*%(%)"'%%]]==], function(c) 16 | return "%" .. tostring(("%x"):format(c:byte()):upper()) 17 | end)) 18 | end 19 | local extend 20 | extend = function(t, ...) 21 | local _list_0 = { 22 | ... 23 | } 24 | for _index_0 = 1, #_list_0 do 25 | local other = _list_0[_index_0] 26 | if other ~= nil then 27 | for k, v in pairs(other) do 28 | t[k] = v 29 | end 30 | end 31 | end 32 | return t 33 | end 34 | local xml_escape 35 | do 36 | local punct = "[%^$()%.%[%]*+%-?]" 37 | local escape_patt 38 | escape_patt = function(str) 39 | return (str:gsub(punct, function(p) 40 | return "%" .. p 41 | end)) 42 | end 43 | local xml_escape_entities = { 44 | ['&'] = '&', 45 | ['<'] = '<', 46 | ['>'] = '>', 47 | ['"'] = '"', 48 | ["'"] = ''' 49 | } 50 | local xml_unescape_entities = { } 51 | for key, value in pairs(xml_escape_entities) do 52 | xml_unescape_entities[value] = key 53 | end 54 | local xml_escape_pattern = "[" .. concat((function() 55 | local _accum_0 = { } 56 | local _len_0 = 1 57 | for char in pairs(xml_escape_entities) do 58 | _accum_0[_len_0] = escape_patt(char) 59 | _len_0 = _len_0 + 1 60 | end 61 | return _accum_0 62 | end)()) .. "]" 63 | xml_escape = function(text) 64 | return (text:gsub(xml_escape_pattern, xml_escape_entities)) 65 | end 66 | end 67 | local LOMFormatter 68 | do 69 | local _class_0 70 | local find_node, filter_nodes, node_value 71 | local _base_0 = { 72 | format = function(self, res, code, headers) 73 | if res == "" then 74 | return code, headers 75 | end 76 | if headers["x-goog-generation"] then 77 | return res 78 | end 79 | res = self.lom.parse(res) 80 | if not res then 81 | return nil, "Failed to parse result " .. tostring(code) 82 | end 83 | if self[res.tag] then 84 | return self[res.tag](self, res) 85 | else 86 | return res, code 87 | end 88 | end, 89 | ["ListAllMyBucketsResult"] = function(self, res) 90 | local buckets_node = find_node(res, "Buckets") 91 | return (function() 92 | local _accum_0 = { } 93 | local _len_0 = 1 94 | for _index_0 = 1, #buckets_node do 95 | local bucket = buckets_node[_index_0] 96 | _accum_0[_len_0] = { 97 | name = node_value(bucket, "Name"), 98 | creation_date = node_value(bucket, "CreationDate") 99 | } 100 | _len_0 = _len_0 + 1 101 | end 102 | return _accum_0 103 | end)() 104 | end, 105 | ["ListBucketResult"] = function(self, res) 106 | return (function() 107 | local _accum_0 = { } 108 | local _len_0 = 1 109 | local _list_0 = filter_nodes(res, "Contents") 110 | for _index_0 = 1, #_list_0 do 111 | local node = _list_0[_index_0] 112 | _accum_0[_len_0] = { 113 | key = node_value(node, "Key"), 114 | size = tonumber(node_value(node, "Size")), 115 | last_modified = node_value(node, "LastModified") 116 | } 117 | _len_0 = _len_0 + 1 118 | end 119 | return _accum_0 120 | end)() 121 | end, 122 | ["Error"] = function(self, res) 123 | return { 124 | error = true, 125 | message = node_value(res, "Message"), 126 | code = node_value(res, "Code"), 127 | details = node_value(res, "Details") 128 | } 129 | end 130 | } 131 | _base_0.__index = _base_0 132 | _class_0 = setmetatable({ 133 | __init = function(self) 134 | self.lom = require("lxp.lom") 135 | end, 136 | __base = _base_0, 137 | __name = "LOMFormatter" 138 | }, { 139 | __index = _base_0, 140 | __call = function(cls, ...) 141 | local _self_0 = setmetatable({}, _base_0) 142 | cls.__init(_self_0, ...) 143 | return _self_0 144 | end 145 | }) 146 | _base_0.__class = _class_0 147 | local self = _class_0 148 | find_node = function(node, tag) 149 | for _index_0 = 1, #node do 150 | local child = node[_index_0] 151 | if child.tag == tag then 152 | return child 153 | end 154 | end 155 | end 156 | filter_nodes = function(node, tag) 157 | return (function() 158 | local _accum_0 = { } 159 | local _len_0 = 1 160 | for _index_0 = 1, #node do 161 | local _continue_0 = false 162 | repeat 163 | local child = node[_index_0] 164 | if not (child.tag == tag) then 165 | _continue_0 = true 166 | break 167 | end 168 | local _value_0 = child 169 | _accum_0[_len_0] = _value_0 170 | _len_0 = _len_0 + 1 171 | _continue_0 = true 172 | until true 173 | if not _continue_0 then 174 | break 175 | end 176 | end 177 | return _accum_0 178 | end)() 179 | end 180 | node_value = function(node, tag) 181 | local child = find_node(node, tag) 182 | return child and child[1] 183 | end 184 | LOMFormatter = _class_0 185 | end 186 | local Bucket 187 | do 188 | local _class_0 189 | local forward_methods 190 | local _base_0 = { } 191 | _base_0.__index = _base_0 192 | _class_0 = setmetatable({ 193 | __init = function(self, bucket_name, storage) 194 | self.bucket_name, self.storage = bucket_name, storage 195 | end, 196 | __base = _base_0, 197 | __name = "Bucket" 198 | }, { 199 | __index = _base_0, 200 | __call = function(cls, ...) 201 | local _self_0 = setmetatable({}, _base_0) 202 | cls.__init(_self_0, ...) 203 | return _self_0 204 | end 205 | }) 206 | _base_0.__class = _class_0 207 | local self = _class_0 208 | forward_methods = { 209 | ["get_bucket"] = "list", 210 | "get_file", 211 | "delete_file", 212 | "head_file", 213 | "put_file", 214 | "put_file_string", 215 | "file_url" 216 | } 217 | for k, v in pairs(forward_methods) do 218 | local name, self_name 219 | if type(k) == "number" then 220 | name, self_name = v, v 221 | else 222 | name, self_name = k, v 223 | end 224 | self.__base[self_name] = function(self, ...) 225 | return self.storage[name](self.storage, self.bucket_name, ...) 226 | end 227 | end 228 | Bucket = _class_0 229 | end 230 | local CloudStorage 231 | do 232 | local _class_0 233 | local _base_0 = { 234 | url_base = "commondatastorage.googleapis.com", 235 | api_base = "storage.googleapis.com", 236 | _headers = function(self) 237 | return { 238 | ["x-goog-api-version"] = 2, 239 | ["x-goog-project-id"] = self.project_id, 240 | ["Authorization"] = "OAuth " .. tostring(self.oauth:get_access_token()), 241 | ["Date"] = date():fmt("${http}") 242 | } 243 | end, 244 | _request = function(self, method, path, data, headers) 245 | if method == nil then 246 | method = "GET" 247 | end 248 | local http = h.get() 249 | local out = { } 250 | local r = { 251 | url = "https://" .. tostring(self.api_base) .. tostring(path), 252 | source = data and ltn12.source.string(data), 253 | method = method, 254 | headers = extend(self:_headers(), headers), 255 | sink = ltn12.sink.table(out) 256 | } 257 | local _, code, res_headers = http.request(r) 258 | local res 259 | res, code = self.formatter:format(table.concat(out), code, res_headers) 260 | if type(res) == "table" and res.error then 261 | return nil, tostring(res.message) .. " " .. tostring(res.details), res 262 | else 263 | return res, code 264 | end 265 | end, 266 | bucket = function(self, bucket) 267 | return Bucket(bucket, self) 268 | end, 269 | file_url = function(self, bucket, key, opts) 270 | return self:bucket_url(bucket, opts) .. "/" .. tostring(key) 271 | end, 272 | bucket_url = function(self, bucket, opts) 273 | if opts == nil then 274 | opts = { } 275 | end 276 | local scheme = opts.scheme or "https" 277 | if opts.subdomain then 278 | return tostring(scheme) .. "://" .. tostring(bucket) .. "." .. tostring(self.url_base) 279 | else 280 | return tostring(scheme) .. "://" .. tostring(self.url_base) .. "/" .. tostring(bucket) 281 | end 282 | end, 283 | get_service = function(self) 284 | return self:_get("/") 285 | end, 286 | get_bucket = function(self, bucket) 287 | return self:_get("/" .. tostring(bucket)) 288 | end, 289 | get_file = function(self, bucket, key, opts) 290 | assert(type(key) == "string" and key ~= "", "Invalid key (missing or empty string)") 291 | return self:_get("/" .. tostring(bucket) .. "/" .. tostring(url.escape(key)), nil, opts and opts.headers) 292 | end, 293 | delete_file = function(self, bucket, key) 294 | assert(type(key) == "string" and key ~= "", "Invalid key for deletion (missing or empty string)") 295 | return self:_delete("/" .. tostring(bucket) .. "/" .. tostring(url.escape(key))) 296 | end, 297 | head_file = function(self, bucket, key) 298 | assert(type(key) == "string" and key ~= "", "Invalid key (missing or empty string)") 299 | return self:_head("/" .. tostring(bucket) .. "/" .. tostring(url.escape(key))) 300 | end, 301 | put_file_acl = function(self, bucket, key, acl) 302 | assert(type(key) == "string" and key ~= "", "Invalid key (missing or empty string)") 303 | return self:_put("/" .. tostring(bucket) .. "/" .. tostring(url.escape(key)) .. "?acl", "", { 304 | ["Content-length"] = 0, 305 | ["x-goog-acl"] = acl 306 | }) 307 | end, 308 | put_file_string = function(self, bucket, key, data, options) 309 | if options == nil then 310 | options = { } 311 | end 312 | assert(not options.key, "key is not an option, but an argument") 313 | if type(data) == "table" then 314 | error("put_file_string interface has changed: key is now the second argument") 315 | end 316 | assert(type(key) == "string" and key ~= "", "Invalid key (missing or empty string)") 317 | assert(type(data) == "string", "expected string for data") 318 | return self:_put("/" .. tostring(bucket) .. "/" .. tostring(url.escape(key)), data, extend({ 319 | ["Content-length"] = #data, 320 | ["Content-type"] = options.mimetype, 321 | ["x-goog-acl"] = options.acl or "public-read" 322 | }, options.headers)) 323 | end, 324 | put_file = function(self, bucket, fname, options) 325 | if options == nil then 326 | options = { } 327 | end 328 | local data 329 | do 330 | local f = io.open(fname) 331 | if f then 332 | do 333 | local _with_0 = f:read("*a") 334 | f:close() 335 | data = _with_0 336 | end 337 | else 338 | data = error("Failed to read file: " .. tostring(fname)) 339 | end 340 | end 341 | options.mimetype = options.mimetype or mimetypes.guess(fname) 342 | local key = options.key or fname 343 | return self:put_file_string(bucket, key, data, options) 344 | end, 345 | copy_file = function(self, source_bucket, source_key, dest_bucket, dest_key, options) 346 | if options == nil then 347 | options = { } 348 | end 349 | return self:_put("/" .. tostring(dest_bucket) .. "/" .. tostring(url.escape(dest_key)), "", extend({ 350 | ["Content-length"] = "0", 351 | ["x-goog-copy-source"] = "/" .. tostring(source_bucket) .. "/" .. tostring(source_key), 352 | ["x-goog-acl"] = options.acl or "public-read", 353 | ["x-goog-metadata-directive"] = options.metadata_directive 354 | }, options.headers)) 355 | end, 356 | compose = function(self, bucket, key, source_keys, options) 357 | if options == nil then 358 | options = { } 359 | end 360 | assert(type(source_keys) == "table" and next(source_keys), "invalid source keys") 361 | local payload_buffer = { 362 | "" 363 | } 364 | for _index_0 = 1, #source_keys do 365 | local key_obj = source_keys[_index_0] 366 | local name, generation, if_generation_match 367 | if type(key_obj) == "table" then 368 | name, generation, if_generation_match = key_obj.name, key_obj.generation, key_obj.if_generation_match 369 | else 370 | name = key_obj 371 | end 372 | assert(name, "missing source key name for compose") 373 | table.insert(payload_buffer, "") 374 | table.insert(payload_buffer, "" .. tostring(xml_escape(name)) .. "") 375 | if generation then 376 | table.insert(payload_buffer, "" .. tostring(xml_escape(generation)) .. "") 377 | end 378 | if if_generation_match then 379 | table.insert(payload_buffer, "" .. tostring(xml_escape(if_generation_match)) .. "") 380 | end 381 | table.insert(payload_buffer, "") 382 | end 383 | table.insert(payload_buffer, "") 384 | local payload = table.concat(payload_buffer) 385 | return self:_put("/" .. tostring(bucket) .. "/" .. tostring(url.escape(key)) .. "?compose", payload, extend({ 386 | ["Content-length"] = #payload, 387 | ["x-goog-acl"] = options.acl or "public-read", 388 | ["Content-type"] = options.mimetype 389 | }, options.headers)) 390 | end, 391 | start_resumable_upload = function(self, bucket, key, options) 392 | if options == nil then 393 | options = { } 394 | end 395 | assert(bucket, "missing bucket") 396 | assert(key, "missing key") 397 | if type(key) == "table" then 398 | options = key 399 | key = assert(options.key, "missing key") 400 | end 401 | return self:_post("/" .. tostring(bucket) .. "/" .. tostring(url.escape(key)), "", extend({ 402 | ["Content-type"] = options.mimetype, 403 | ["Content-length"] = 0, 404 | ["x-goog-acl"] = options.acl or "public-read", 405 | ["x-goog-resumable"] = "start" 406 | }, options.headers)) 407 | end, 408 | canonicalize_headers = function(self, headers) 409 | local header_pairs 410 | do 411 | local _accum_0 = { } 412 | local _len_0 = 1 413 | for k, v in pairs(headers) do 414 | _accum_0[_len_0] = { 415 | k:lower(), 416 | v 417 | } 418 | _len_0 = _len_0 + 1 419 | end 420 | header_pairs = _accum_0 421 | end 422 | do 423 | local _accum_0 = { } 424 | local _len_0 = 1 425 | for _index_0 = 1, #header_pairs do 426 | local e = header_pairs[_index_0] 427 | if (e[1]:match("x%-goog.*") and not e[1]:match("x%-goog%-encryption%-key.*")) then 428 | _accum_0[_len_0] = e 429 | _len_0 = _len_0 + 1 430 | end 431 | end 432 | header_pairs = _accum_0 433 | end 434 | table.sort(header_pairs, function(a, b) 435 | return a[1] < b[1] 436 | end) 437 | local values 438 | do 439 | local _accum_0 = { } 440 | local _len_0 = 1 441 | for _index_0 = 1, #header_pairs do 442 | local e = header_pairs[_index_0] 443 | _accum_0[_len_0] = e[1] .. ":" .. e[2]:gsub("\r?\n", " ") 444 | _len_0 = _len_0 + 1 445 | end 446 | values = _accum_0 447 | end 448 | return concat(values, "\n") 449 | end, 450 | encode_and_sign_policy = function(self, expiration, conditions) 451 | if type(expiration) == "number" then 452 | expiration = os.date("!%Y-%m-%dT%H:%M:%SZ", expiration) 453 | end 454 | local doc = mime.b64([[{"expiration":]] .. json.encode(expiration) .. [[,"conditions":]] .. json.encode(conditions) .. [[}]]) 455 | return doc, self.oauth:sign_string(doc) 456 | end, 457 | signed_url = function(self, bucket, key, expiration, opts) 458 | if opts == nil then 459 | opts = { } 460 | end 461 | local headers, verb, scheme 462 | headers, verb, scheme = opts.headers, opts.verb, opts.scheme 463 | verb = verb or "GET" 464 | scheme = scheme or "https" 465 | key = url_encode_key(key) 466 | local path = "/" .. tostring(bucket) .. "/" .. tostring(key) 467 | expiration = tostring(expiration) 468 | local elements = { 469 | verb, 470 | "", 471 | "", 472 | expiration 473 | } 474 | if headers and next(headers) then 475 | table.insert(elements, self:canonicalize_headers(headers)) 476 | end 477 | table.insert(elements, "") 478 | local str = concat(elements, "\n") 479 | str = str .. path 480 | local signature = self.oauth:sign_string(str) 481 | local escape 482 | escape = function(str) 483 | return (str:gsub("[/+]", { 484 | ["+"] = "%2B", 485 | ["/"] = "%2F" 486 | })) 487 | end 488 | return concat({ 489 | scheme, 490 | "://", 491 | self.url_base, 492 | path, 493 | "?GoogleAccessId=", 494 | self.oauth.client_email, 495 | "&Expires=", 496 | expiration, 497 | "&Signature=", 498 | escape(signature) 499 | }) 500 | end, 501 | upload_url = function(self, bucket, key, opts) 502 | if opts == nil then 503 | opts = { } 504 | end 505 | local content_disposition, filename, acl, success_action_redirect, expires, size_limit, scheme 506 | content_disposition, filename, acl, success_action_redirect, expires, size_limit, scheme = opts.content_disposition, opts.filename, opts.acl, opts.success_action_redirect, opts.expires, opts.size_limit, opts.scheme 507 | expires = expires or (os.time() + 60 ^ 2) 508 | acl = acl or "project-private" 509 | if filename then 510 | content_disposition = content_disposition or "attachment" 511 | local filename_quoted = filename:gsub('"', "\\%1") 512 | content_disposition = content_disposition .. "; filename=\"" .. tostring(filename_quoted) .. "\"" 513 | end 514 | local policy = { } 515 | insert(policy, { 516 | acl = acl 517 | }) 518 | insert(policy, { 519 | bucket = bucket 520 | }) 521 | insert(policy, { 522 | "eq", 523 | "$key", 524 | key 525 | }) 526 | if content_disposition then 527 | insert(policy, { 528 | "eq", 529 | "$Content-Disposition", 530 | content_disposition 531 | }) 532 | end 533 | if size_limit then 534 | insert(policy, { 535 | "content-length-range", 536 | 0, 537 | size_limit 538 | }) 539 | end 540 | if success_action_redirect then 541 | insert(policy, { 542 | success_action_redirect = success_action_redirect 543 | }) 544 | end 545 | local signature 546 | policy, signature = self:encode_and_sign_policy(expires, policy) 547 | local action = self:bucket_url(bucket, { 548 | subdomain = true, 549 | scheme = scheme 550 | }) 551 | local params = { 552 | acl = acl, 553 | policy = policy, 554 | signature = signature, 555 | key = key, 556 | success_action_redirect = success_action_redirect, 557 | ["Content-Disposition"] = content_disposition, 558 | GoogleAccessId = self.oauth.client_email 559 | } 560 | return action, params 561 | end 562 | } 563 | _base_0.__index = _base_0 564 | _class_0 = setmetatable({ 565 | __init = function(self, oauth, project_id) 566 | self.oauth, self.project_id = oauth, project_id 567 | self.formatter = LOMFormatter() 568 | end, 569 | __base = _base_0, 570 | __name = "CloudStorage" 571 | }, { 572 | __index = _base_0, 573 | __call = function(cls, ...) 574 | local _self_0 = setmetatable({}, _base_0) 575 | cls.__init(_self_0, ...) 576 | return _self_0 577 | end 578 | }) 579 | _base_0.__class = _class_0 580 | local self = _class_0 581 | self.from_json_key_file = function(self, file) 582 | local file_contents = assert(assert(io.open(file)):read("*a")) 583 | json = require("cjson") 584 | local obj = assert(json.decode(file_contents)) 585 | local OAuth 586 | OAuth = require("cloud_storage.oauth").OAuth 587 | local oauth = OAuth(obj.client_email) 588 | oauth:_load_private_key(obj.private_key) 589 | return CloudStorage(oauth, obj.client_id) 590 | end 591 | local _list_0 = { 592 | "GET", 593 | "POST", 594 | "PUT", 595 | "DELETE", 596 | "HEAD" 597 | } 598 | for _index_0 = 1, #_list_0 do 599 | local m = _list_0[_index_0] 600 | self.__base["_" .. tostring(m:lower())] = function(self, ...) 601 | return self:_request(m, ...) 602 | end 603 | end 604 | CloudStorage = _class_0 605 | end 606 | return { 607 | CloudStorage = CloudStorage, 608 | Bucket = Bucket, 609 | url_encode_key = url_encode_key 610 | } 611 | -------------------------------------------------------------------------------- /spec/cloud_storage_spec.moon: -------------------------------------------------------------------------------- 1 | 2 | oauth = require "cloud_storage.oauth" 3 | google = require "cloud_storage.google" 4 | 5 | TEST_KEY_PATH = "spec/test_key.pem" 6 | TEST_KEY_PATH_JSON = "spec/test_key.json" 7 | 8 | describe "cloud_storage", -> 9 | it "should create an oauth", -> 10 | o = oauth.OAuth "leaf@leafo.net", TEST_KEY_PATH 11 | storage = google.CloudStorage o, "111111111111" 12 | 13 | describe "oauth", -> 14 | local o 15 | before_each -> 16 | o = oauth.OAuth "leaf@leafo.net", TEST_KEY_PATH 17 | 18 | it "should make jwt", -> 19 | assert.truthy o\_make_jwt o.client_email, o.private_key 20 | 21 | describe "private key loading", -> 22 | it "should load private key from file", -> 23 | key = o\_private_key! 24 | assert.truthy key 25 | 26 | it "should load private key from string", -> 27 | key_content = assert(io.open(TEST_KEY_PATH))\read "*a" 28 | o\_load_private_key key_content 29 | key = o\_private_key! 30 | assert.truthy key 31 | 32 | it "should fail with invalid key file", -> 33 | bad_oauth = oauth.OAuth "leaf@leafo.net", "nonexistent.pem" 34 | assert.has_error( 35 | -> bad_oauth\_private_key! 36 | ) 37 | 38 | -- invalid file type 39 | bad_oauth = oauth.OAuth "leaf@leafo.net", "spec/cloud_storage_spec.moon" 40 | assert.has_error( 41 | -> bad_oauth\_private_key! 42 | ) 43 | 44 | describe "string signing", -> 45 | -- this should be deterministic since the key is stored in the test suite 46 | it "should sign string and return base64", -> 47 | signature = o\sign_string "test string" 48 | assert.same "Z1x1HRvpY9tWnf+O1HU3D+A7VNHY4LTem4YUORBS6r4rrbYjYhgUntEy9hfwoPeFyilBY4K6mGYctUBBnRAybcqWWDz68rmS0zR3ROy/pBfFGrcbRoFVwQnx/nVliqUH6+i3iPUE/S7haPh6b8O0yy3ltZBhuAYfZAinJiS4mVM=", signature 49 | 50 | it "should produce consistent signatures", -> 51 | sig1 = o\sign_string "consistent test" 52 | sig2 = o\sign_string "consistent test" 53 | assert.same sig1, sig2 54 | 55 | describe "token management", -> 56 | local http_requests 57 | local snapshot 58 | 59 | json = require "cjson" 60 | 61 | before_each -> 62 | snapshot = assert\snapshot! 63 | http_requests = {} 64 | 65 | http = require("cloud_storage.http") 66 | stub(http, "get", { 67 | request: (url, params) -> 68 | table.insert http_requests, {url: url, params: params} 69 | return json.encode { 70 | expires_in: 3600 71 | access_token: "mock-access-token-123" 72 | } 73 | }) 74 | 75 | after_each -> 76 | snapshot\revert! 77 | 78 | it "should refresh access token", -> 79 | token = o\refresh_access_token! 80 | assert.same "mock-access-token-123", token 81 | assert.same "mock-access-token-123", o.access_token 82 | assert.truthy o.expires_at 83 | 84 | it "should make proper JWT request format", -> 85 | o\refresh_access_token! 86 | assert.same 1, #http_requests 87 | request = http_requests[1] 88 | assert.same o.auth_url, request.url 89 | assert.truthy request.params\find "grant_type" 90 | assert.truthy request.params\find "assertion" 91 | 92 | it "should cache tokens and not refresh when valid", -> 93 | o.access_token = "existing-token" 94 | o.expires_at = os.time! + 1000 95 | 96 | token = o\get_access_token! 97 | assert.same "existing-token", token 98 | assert.same 0, #http_requests 99 | 100 | it "should refresh expired tokens", -> 101 | o.access_token = "old-token" 102 | o.expires_at = os.time! - 100 103 | 104 | token = o\get_access_token! 105 | assert.same "mock-access-token-123", token 106 | assert.same 1, #http_requests 107 | 108 | describe "with auth errors", -> 109 | before_each -> 110 | http = require("cloud_storage.http") 111 | stub(http, "get", { 112 | request: (url, params) -> 113 | table.insert http_requests, {url: url, params: params} 114 | return json.encode { error: "invalid_grant" } 115 | }) 116 | 117 | it "should handle auth errors", -> 118 | assert.has_error( 119 | -> o\refresh_access_token! 120 | "Failed auth: invalid_grant" 121 | ) 122 | 123 | describe "jwt creation", -> 124 | it "should create valid jwt format", -> 125 | jwt = o\_make_jwt o.client_email, o\_private_key! 126 | parts = [part for part in jwt\gmatch "[^%.]+"] 127 | assert.same 3, #parts 128 | 129 | it "should include proper claims", -> 130 | jwt = o\_make_jwt o.client_email, o\_private_key! 131 | header_b64, claims_b64, sig = jwt\match "([^%.]+)%.([^%.]+)%.([^%.]+)" 132 | 133 | mime = require "mime" 134 | json = require "cjson" 135 | 136 | assert.truthy claims_b64, "claims part should be present" 137 | claims_str = mime.unb64 claims_b64 138 | assert.truthy claims_str, "decoded claims should not be nil" 139 | claims = json.decode claims_str 140 | assert.same o.client_email, claims.iss 141 | assert.same o.auth_url, claims.aud 142 | assert.same o.scope.full_control, claims.scope 143 | assert.truthy claims.iat 144 | assert.truthy claims.exp 145 | 146 | describe "with storage", -> 147 | local storage 148 | 149 | before_each -> 150 | o = oauth.OAuth "leaf@leafo.net", TEST_KEY_PATH 151 | storage = google.CloudStorage o, "111111111111" 152 | 153 | it "generates bucket url", -> 154 | assert.same "https://commondatastorage.googleapis.com/my-bucket", storage\bucket_url "my-bucket" 155 | assert.same "http://my-bucket.commondatastorage.googleapis.com", storage\bucket_url "my-bucket", { 156 | scheme: "http" 157 | subdomain: true 158 | } 159 | 160 | it "generates file url", -> 161 | assert.same "https://commondatastorage.googleapis.com/my-bucket/pics/leafo.png", storage\file_url "my-bucket", "pics/leafo.png" 162 | assert.same "http://my-bucket.commondatastorage.googleapis.com/pics/leafo.png", storage\file_url "my-bucket", "pics/leafo.png", { 163 | scheme: "http" 164 | subdomain: true 165 | } 166 | 167 | it "should create signed url", -> 168 | url = storage\signed_url "thebucket", "hello.txt", 10000 169 | assert.same "https://commondatastorage.googleapis.com/thebucket/hello.txt?GoogleAccessId=leaf@leafo.net&Expires=10000&Signature=W8kzLHy1p0wAEjR%2FdPb9VeJ%2B%2Bm154%2BEJFBo47vdWmVGNsFFDo6n%2Bhnpy17bYQH9xF8H2lABp%2BJyn%2B0ViJimIDZwiQ%2FtPe1bTTrXVA1Uzucu7tdH29M60mnwRCyxYKQqoVkDhwki1HuUPluRRVndkrdfU1J8Cq8qIEaXcGDzt3O4=", url 170 | 171 | it "should create signed url with options", -> 172 | url = storage\signed_url "thebucket", "hello.txt", 10000, { 173 | headers: { 174 | "Content-Disposition": "attachment" -- this header is ignored 175 | "x-goog-resumable": "start" 176 | } 177 | verb: "POST" 178 | scheme: "http" 179 | } 180 | assert.same 'http://commondatastorage.googleapis.com/thebucket/hello.txt?GoogleAccessId=leaf@leafo.net&Expires=10000&Signature=GwFHuaLI48MuvwD7YsoPlF3TMe1oZg1hFjdb37pzw65HKtNshW87gzCY7rXjYX4HmFr%2FYHJKZwQ4WQo30IGYYjG9ccJPAJaySYUW7JWkrk34h%2BlWYyhX0kq8ayEnCL3y96UJc3%2F0oizsUoIxxPek6KyzaWxEENWQQQVRxD6q2g0=', url 181 | 182 | it "should encode file with funky chars in it", -> 183 | url = storage\signed_url "thebucket", "he[f]llo#one.txt", 10000 184 | assert.falsy url\find "#", 1, true 185 | assert.falsy url\find "]", 1, true 186 | assert.falsy url\find "[", 1, true 187 | 188 | it "should encode even more chars", -> 189 | import url_encode_key from require "cloud_storage.google" 190 | assert.same [[%21_@_$_%5E_%2A_%28_%29_+_=_%5D_%5B_\_/_._,_%27_%22_%25]], 191 | url_encode_key [[!_@_$_^_*_(_)_+_=_]_[_\_/_._,_'_"_%]] 192 | 193 | it "should canonicalize headers", -> 194 | headers = {} 195 | headers["x-goog-acl"] = "project-private" 196 | headers["x-goog-meta-hello"] = "hella\nhelli" 197 | headers["x-goog-encryption-key"] = "best" 198 | headers["x-goog-encryption-key-sha256"] = "dad" 199 | headers["Content-Disposition"] = "attachment" 200 | headers["Content-Length"] = 0 201 | assert.same "x-goog-acl:project-private\nx-goog-meta-hello:hella helli", storage\canonicalize_headers headers 202 | 203 | it "creates upload url", -> 204 | url, params = assert storage\upload_url "thebucket", "hello.txt", { 205 | expires: 10000 206 | filename: "bart.zip" 207 | size_limit: 1024 208 | acl: "public-read" 209 | success_action_redirect: "http://leafo.net" 210 | } 211 | 212 | assert.same "https://thebucket.commondatastorage.googleapis.com", url 213 | 214 | assert.same { 215 | "Content-Disposition": "attachment; filename=\"bart.zip\"", 216 | GoogleAccessId: "leaf@leafo.net", 217 | signature: "BtimAyE8GUOcCRE3ie7/6AjAuVXn/urTro69vhMB35oOPzlWT23iguL9mi2D7KQ0kAP+6uJL9u3Dr7xtLgMhMFDFWje9GZ9VdZlEBELjyB+MWrXZm1fvMcbr8WfWAK/JCezEe3keOdXpD5w5kV6lydVKZWVapUNf0u2CD1WtCG0=", 218 | success_action_redirect: "http://leafo.net", 219 | policy: "eyJleHBpcmF0aW9uIjoiMTk3MC0wMS0wMVQwMjo0Njo0MFoiLCJjb25kaXRpb25zIjpbeyJhY2wiOiJwdWJsaWMtcmVhZCJ9LHsiYnVja2V0IjoidGhlYnVja2V0In0sWyJlcSIsIiRrZXkiLCJoZWxsby50eHQiXSxbImVxIiwiJENvbnRlbnQtRGlzcG9zaXRpb24iLCJhdHRhY2htZW50OyBmaWxlbmFtZT1cImJhcnQuemlwXCIiXSxbImNvbnRlbnQtbGVuZ3RoLXJhbmdlIiwwLDEwMjRdLHsic3VjY2Vzc19hY3Rpb25fcmVkaXJlY3QiOiJodHRwOlwvXC9sZWFmby5uZXQifV19", 220 | key:"hello.txt", 221 | acl: "public-read" 222 | }, params 223 | 224 | mime = require "mime" 225 | json = require "cjson" 226 | 227 | policy = json.decode (mime.unb64 params.policy) 228 | assert.same { 229 | expiration: "1970-01-01T02:46:40Z" 230 | conditions: { 231 | { acl: "public-read" } 232 | { bucket: "thebucket" } 233 | { "eq", "$key", "hello.txt" } 234 | { "eq", "$Content-Disposition", "attachment; filename=\"bart.zip\"" } 235 | { "content-length-range", 0, 1024 } 236 | { success_action_redirect: "http://leafo.net" } 237 | } 238 | }, policy 239 | 240 | describe "with http", -> 241 | local http_requests 242 | local snapshot 243 | 244 | json = require "cjson" 245 | 246 | before_each -> 247 | snapshot = assert\snapshot! 248 | 249 | after_each -> 250 | snapshot\revert! 251 | 252 | before_each -> 253 | http_requests = {} 254 | 255 | http = require("cloud_storage.http") 256 | stub(http, "get", { 257 | request: (r) -> 258 | -- let the token request go through 259 | if r == "https://accounts.google.com/o/oauth2/token" 260 | return json.encode { 261 | expires_in: 100000 262 | access_token: "my-fake-access-token" 263 | } 264 | 265 | dupe = {k,v for k,v in pairs r} 266 | if dupe.source 267 | dupe.source = dupe.source! 268 | dupe.sink = nil 269 | dupe.headers.Date = nil 270 | table.insert http_requests, dupe 271 | }) 272 | 273 | it "put_file_string", -> 274 | storage\put_file_string "mybucket", "hello.txt", "the contents", { 275 | acl: "public-read" 276 | } 277 | 278 | assert.same http_requests, { 279 | { 280 | url: "https://storage.googleapis.com/mybucket/hello%2etxt" 281 | method: "PUT" 282 | source: "the contents" 283 | headers: { 284 | "Content-length": 12 285 | "x-goog-acl": "public-read" 286 | "x-goog-api-version": 2 287 | "x-goog-project-id": "111111111111" 288 | Authorization: "OAuth my-fake-access-token" 289 | } 290 | } 291 | } 292 | 293 | 294 | it "get_bucket", -> 295 | storage\get_bucket "mybucket" 296 | assert.same { 297 | { 298 | url: "https://storage.googleapis.com/mybucket" 299 | method: "GET" 300 | headers: { 301 | "x-goog-api-version": 2 302 | "x-goog-project-id": "111111111111" 303 | Authorization: "OAuth my-fake-access-token" 304 | } 305 | } 306 | }, http_requests 307 | 308 | it "get_file", -> 309 | storage\get_file "mybucket", "source/my-file.lua" 310 | assert.same { 311 | { 312 | url: "https://storage.googleapis.com/mybucket/source%2fmy%2dfile%2elua" 313 | method: "GET" 314 | headers: { 315 | "x-goog-api-version": 2 316 | "x-goog-project-id": "111111111111" 317 | Authorization: "OAuth my-fake-access-token" 318 | } 319 | } 320 | }, http_requests 321 | 322 | it "get_file fails with empty key", -> 323 | assert.has_error( 324 | -> storage\get_file "mybucket", "" 325 | "Invalid key (missing or empty string)" 326 | ) 327 | assert.same { }, http_requests 328 | 329 | it "get_file fails with missing key", -> 330 | assert.has_error( 331 | -> storage\get_file "mybucket" 332 | "Invalid key (missing or empty string)" 333 | ) 334 | assert.same { }, http_requests 335 | 336 | it "copy_file", -> 337 | storage\copy_file "from_bucket", "input/a.txt", "to_bucket", "output/b.txt", { 338 | acl: "private" 339 | } 340 | 341 | assert.same { 342 | { 343 | url: "https://storage.googleapis.com/to_bucket/output%2fb%2etxt" 344 | method: "PUT" 345 | headers: { 346 | "Content-length": "0" 347 | "x-goog-api-version": 2 348 | "x-goog-acl": "private" 349 | "x-goog-copy-source": "/from_bucket/input/a.txt" 350 | "x-goog-project-id": "111111111111" 351 | Authorization: "OAuth my-fake-access-token" 352 | } 353 | } 354 | }, http_requests 355 | 356 | it "delete_file", -> 357 | storage\delete_file "mybucket", "source/my-pic..png" 358 | assert.same { 359 | { 360 | url: "https://storage.googleapis.com/mybucket/source%2fmy%2dpic%2e%2epng" 361 | method: "DELETE" 362 | headers: { 363 | "x-goog-api-version": 2 364 | "x-goog-project-id": "111111111111" 365 | Authorization: "OAuth my-fake-access-token" 366 | } 367 | } 368 | }, http_requests 369 | 370 | it "delete_file fails with empty string key", -> 371 | assert.has_error( 372 | -> storage\delete_file "mybucket", "" 373 | "Invalid key for deletion (missing or empty string)" 374 | ) 375 | 376 | assert.same {}, http_requests 377 | 378 | it "delete_file fails with missing string key", -> 379 | assert.has_error( 380 | -> storage\delete_file "mybucket" 381 | "Invalid key for deletion (missing or empty string)" 382 | ) 383 | 384 | assert.same {}, http_requests 385 | 386 | it "get_service", -> 387 | storage\get_service! 388 | assert.same { 389 | { 390 | url: "https://storage.googleapis.com/" 391 | method: "GET" 392 | headers: { 393 | "x-goog-api-version": 2 394 | "x-goog-project-id": "111111111111" 395 | Authorization: "OAuth my-fake-access-token" 396 | } 397 | } 398 | }, http_requests 399 | 400 | it "head_file", -> 401 | storage\head_file "mybucket", "test/file.txt" 402 | assert.same { 403 | { 404 | url: "https://storage.googleapis.com/mybucket/test%2ffile%2etxt" 405 | method: "HEAD" 406 | headers: { 407 | "x-goog-api-version": 2 408 | "x-goog-project-id": "111111111111" 409 | Authorization: "OAuth my-fake-access-token" 410 | } 411 | } 412 | }, http_requests 413 | 414 | it "head_file fails with empty key", -> 415 | assert.has_error( 416 | -> storage\head_file "mybucket", "" 417 | "Invalid key (missing or empty string)" 418 | ) 419 | assert.same {}, http_requests 420 | 421 | it "put_file_acl", -> 422 | storage\put_file_acl "mybucket", "test.txt", "private" 423 | assert.same { 424 | { 425 | url: "https://storage.googleapis.com/mybucket/test%2etxt?acl" 426 | method: "PUT" 427 | headers: { 428 | "Content-length": 0 429 | "x-goog-acl": "private" 430 | "x-goog-api-version": 2 431 | "x-goog-project-id": "111111111111" 432 | Authorization: "OAuth my-fake-access-token" 433 | } 434 | } 435 | }, http_requests 436 | 437 | it "put_file_acl fails with empty key", -> 438 | assert.has_error( 439 | -> storage\put_file_acl "mybucket", "", "private" 440 | "Invalid key (missing or empty string)" 441 | ) 442 | assert.same {}, http_requests 443 | 444 | it "start_resumable_upload", -> 445 | storage\start_resumable_upload "mybucket", "large-file.dat", { 446 | mimetype: "application/octet-stream" 447 | acl: "public-read" 448 | } 449 | assert.same { 450 | { 451 | url: "https://storage.googleapis.com/mybucket/large%2dfile%2edat" 452 | method: "POST" 453 | headers: { 454 | "Content-type": "application/octet-stream" 455 | "Content-length": 0 456 | "x-goog-acl": "public-read" 457 | "x-goog-resumable": "start" 458 | "x-goog-api-version": 2 459 | "x-goog-project-id": "111111111111" 460 | Authorization: "OAuth my-fake-access-token" 461 | } 462 | } 463 | }, http_requests 464 | 465 | it "start_resumable_upload with key in options", -> 466 | storage\start_resumable_upload "mybucket", { 467 | key: "alternate.dat" 468 | acl: "private" 469 | } 470 | assert.same { 471 | { 472 | url: "https://storage.googleapis.com/mybucket/alternate%2edat" 473 | method: "POST" 474 | headers: { 475 | "Content-length": 0 476 | "x-goog-acl": "private" 477 | "x-goog-resumable": "start" 478 | "x-goog-api-version": 2 479 | "x-goog-project-id": "111111111111" 480 | Authorization: "OAuth my-fake-access-token" 481 | } 482 | } 483 | }, http_requests 484 | 485 | it "compose files", -> 486 | storage\compose "mybucket", "composed.txt", {"part1.txt", "part2.txt"}, { 487 | acl: "public-read" 488 | } 489 | assert.same 1, #http_requests 490 | request = http_requests[1] 491 | assert.same "https://storage.googleapis.com/mybucket/composed%2etxt?compose", request.url 492 | assert.same "PUT", request.method 493 | assert.truthy request.source\find "" 494 | assert.truthy request.source\find "part1.txt" 495 | assert.truthy request.source\find "part2.txt" 496 | assert.same "public-read", request.headers["x-goog-acl"] 497 | 498 | it "compose files with generation", -> 499 | storage\compose "mybucket", "composed.txt", { 500 | {name: "part1.txt", generation: "123"} 501 | "part2.txt" 502 | } 503 | request = http_requests[1] 504 | assert.truthy request.source\find "123" 505 | assert.truthy request.source\find "part1.txt" 506 | assert.truthy request.source\find "part2.txt" 507 | 508 | it "encode_and_sign_policy", -> 509 | conditions = { 510 | {"eq", "$key", "test.txt"} 511 | {"content-length-range", 0, 1024} 512 | } 513 | policy, signature = storage\encode_and_sign_policy 1000, conditions 514 | 515 | assert.truthy policy 516 | assert.truthy signature 517 | 518 | mime = require "mime" 519 | json = require "cjson" 520 | 521 | decoded_policy_str = mime.unb64 policy 522 | assert.truthy decoded_policy_str 523 | decoded_policy = json.decode decoded_policy_str 524 | assert.same "1970-01-01T00:16:40Z", decoded_policy.expiration 525 | assert.same conditions, decoded_policy.conditions 526 | 527 | it "encode_and_sign_policy with string expiration", -> 528 | conditions = {{"eq", "$key", "test.txt"}} 529 | policy, signature = storage\encode_and_sign_policy "2025-01-01T00:00:00Z", conditions 530 | 531 | mime = require "mime" 532 | json = require "cjson" 533 | 534 | decoded_policy_str = mime.unb64 policy 535 | assert.truthy decoded_policy_str 536 | decoded_policy = json.decode decoded_policy_str 537 | assert.same "2025-01-01T00:00:00Z", decoded_policy.expiration 538 | 539 | it "put_file reads from filesystem", -> 540 | -- Store original io.open 541 | original_io_open = io.open 542 | stub(io, "open", (filename, mode) -> 543 | if filename == "test_upload.txt" 544 | return { 545 | read: (self, format) -> "test file content" 546 | close: -> 547 | } 548 | else 549 | original_io_open filename, mode -- use original for other files 550 | ) 551 | 552 | -- Test the basic functionality without key override 553 | storage\put_file "mybucket", "test_upload.txt", { 554 | acl: "private" 555 | } 556 | 557 | assert.same 1, #http_requests 558 | request = http_requests[1] 559 | assert.same "https://storage.googleapis.com/mybucket/test_upload%2etxt", request.url 560 | assert.same "PUT", request.method 561 | assert.same "test file content", request.source 562 | assert.same "private", request.headers["x-goog-acl"] 563 | assert.same "text/plain", request.headers["Content-type"] 564 | 565 | it "put_file uses filename as key by default", -> 566 | -- Store original io.open 567 | original_io_open = io.open 568 | stub(io, "open", (filename, mode) -> 569 | if filename == "test_default.txt" 570 | return { 571 | read: (self, format) -> "default key test" 572 | close: -> 573 | } 574 | else 575 | original_io_open filename, mode -- use original for other files 576 | ) 577 | 578 | storage\put_file "mybucket", "test_default.txt" 579 | 580 | request = http_requests[1] 581 | assert.same "https://storage.googleapis.com/mybucket/test_default%2etxt", request.url 582 | assert.same "default key test", request.source 583 | 584 | it "put_file fails with missing file", -> 585 | stub(io, "open", -> nil) -- simulate file not found 586 | 587 | assert.has_error( 588 | -> storage\put_file "mybucket", "nonexistent.txt" 589 | "Failed to read file: nonexistent.txt" 590 | ) 591 | assert.same {}, http_requests 592 | 593 | describe "with storage from json key", -> 594 | local storage 595 | 596 | before_each -> 597 | storage = google.CloudStorage\from_json_key_file TEST_KEY_PATH_JSON 598 | 599 | it "should create signed url", -> 600 | url = storage\signed_url "thebucket", "hello.txt", 10000 601 | assert.same "https://commondatastorage.googleapis.com/thebucket/hello.txt?GoogleAccessId=leaf@leafo.net&Expires=10000&Signature=W8kzLHy1p0wAEjR%2FdPb9VeJ%2B%2Bm154%2BEJFBo47vdWmVGNsFFDo6n%2Bhnpy17bYQH9xF8H2lABp%2BJyn%2B0ViJimIDZwiQ%2FtPe1bTTrXVA1Uzucu7tdH29M60mnwRCyxYKQqoVkDhwki1HuUPluRRVndkrdfU1J8Cq8qIEaXcGDzt3O4=", url 602 | --------------------------------------------------------------------------------