├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── handle.go ├── handle_test.go ├── internal └── token.go ├── main_test.go ├── object.go ├── object_test.go ├── server ├── main_test.go ├── server.go └── server_test.go ├── storage.go └── storage_test.go /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: go 3 | go: 4 | - 1.11 # Latest GAE standard 5 | env: 6 | global: 7 | - GAE_SDK_URL=https://storage.googleapis.com/appengine-sdks/featured/go_appengine_sdk_linux_amd64-1.9.68.zip 8 | - PATH=$HOME/go_appengine:$PATH 9 | cache: 10 | directories: 11 | - "$HOME/go_appengine" 12 | 13 | install: 14 | - "[ -f $HOME/go_appengine/goapp ] || (curl -sSLo $HOME/sdk.zip $GAE_SDK_URL && unzip -q -d $HOME $HOME/sdk.zip)" 15 | - goapp get -d google.golang.org/appengine 16 | - goapp get -d ./... 17 | 18 | script: 19 | - goapp test -v ./... 20 | - go vet ./... 21 | - "go get golang.org/x/lint/golint && golint ./..." 22 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Want to contribute? Great! First, read this page (including the small print at the end). 2 | 3 | ### Before you contribute 4 | Before we can use your code, you must sign the 5 | [Google Individual Contributor License Agreement] 6 | (https://cla.developers.google.com/about/google-individual) 7 | (CLA), which you can do online. The CLA is necessary mainly because you own the 8 | copyright to your changes, even after your contribution becomes part of our 9 | codebase, so we need your permission to use and distribute your code. We also 10 | need to be sure of various other things—for instance that you'll tell us if you 11 | know that your code infringes on other people's patents. You don't have to sign 12 | the CLA until after you've submitted your code for review and a member has 13 | approved it, but you must do it before we can put your code into our codebase. 14 | Before you start working on a larger contribution, you should get in touch with 15 | us first through the issue tracker with your idea so that we can help out and 16 | possibly guide you. Coordinating up front makes it much easier to avoid 17 | frustration later on. 18 | 19 | ### Code reviews 20 | All submissions, including submissions by project members, require review. We 21 | use Github pull requests for this purpose. 22 | 23 | ### The small print 24 | Contributions made by corporations are covered by a different agreement than 25 | the one above, the 26 | [Software Grant and Corporate Contributor License Agreement] 27 | (https://cla.developers.google.com/about/google-corporate). 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2015 Google Inc 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # yummy-weasel 2 | 3 | A simple frontend (App Engine app) that serves content from a Google 4 | Cloud Storage (GCS) bucket, while allowing for: 5 | 6 | - HTTPS on custom domain 7 | - support for Service Worker (works only over HTTPS) 8 | - HTTP/2 push 9 | - more robust redirect from naked custom domain 10 | - serving directly from naked custom domain 11 | - keep the deployment/publishing flow using just the existing GCS bucket, 12 | which has proven to be fast and reliable, with partial content updates. 13 | - dynamic server-side logic 14 | 15 | ## design 16 | 17 | The design is simple. Suppose you have a static website www.example.com, 18 | served directly from a GCS gs://www.example.com. 19 | 20 | A very simplified picture of the serving path can be shown as follows. 21 | 22 | +-----+ GET http://www.example.com/dir/index.html 23 | | GCS | <-----------------------------------------+ visitor 24 | +-----+ :-/ 25 | 26 | Weasel enhancement consists of modifying the serving path to: 27 | 28 | +-----+ (2) GET gs://www.example.com/dir/index.html 29 | | GCS | <------------------------------+--------+ 30 | +-----+ | yummy | 31 | | weasel | 32 | +--------+ 33 | ↑ | 34 | (1) | | (3) 35 | GET https://www.example.com/dir/ | ↓ Link: ; rel=preload 36 | 37 | visitor 38 | :-) 39 | 40 | 1. A visitor requests the website page. Note that /index.html is now optional. 41 | Due to GCS restrictions, such suffixes were previously required for 42 | sub-folders, but with this approach they no longer need to be specified. 43 | Also, requests can (and will) be made over HTTPS. 44 | 45 | 2. Weasel fetches the object content from the original GCS bucket and caches 46 | it locally. This step is necessary only if the object hasn't been cached 47 | already or the cache has expired. Cache expiration and invalidation is 48 | based on GCS object cache-control header settings. 49 | 50 | 3. Weasel responds with the GCS object contents. Note that we can optionally 51 | [push](https://w3c.github.io/preload/) additional assets related to the 52 | requested file by using `Link: ; rel=preload` header supported 53 | by GFE. 54 | 55 | 56 | ## license 57 | 58 | Apache License 2.0. 59 | 60 | This is not an official Google product. 61 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/google/weasel 2 | 3 | go 1.13 4 | 5 | require ( 6 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 7 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 8 | google.golang.org/appengine v1.6.3 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= 2 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 3 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 4 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 5 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 6 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 7 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 8 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 9 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 10 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 11 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 12 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 13 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= 14 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 15 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= 16 | golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 17 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 18 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 19 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 20 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 21 | golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 22 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 23 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 24 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 25 | golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 26 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 27 | google.golang.org/appengine v1.6.3 h1:hvZejVcIxAKHR8Pq2gXaDggf6CWT1QEqO+JEBeOKCG8= 28 | google.golang.org/appengine v1.6.3/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= 29 | -------------------------------------------------------------------------------- /handle.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package weasel 16 | 17 | import ( 18 | "encoding/json" 19 | "io" 20 | "net/http" 21 | "sort" 22 | "strings" 23 | 24 | "google.golang.org/appengine" 25 | "google.golang.org/appengine/log" 26 | ) 27 | 28 | // allowMethods is a comman-separated list of allowed HTTP methods, 29 | // suitable for Allow or CORS allow-methods header. 30 | var allowMethods = "GET, HEAD, OPTIONS" 31 | 32 | // ServeObject writes object o to w, with optional body and CORS headers, 33 | // based on the in-flight request r. 34 | func (s *Storage) ServeObject(w http.ResponseWriter, r *http.Request, o *Object) error { 35 | // headers 36 | h := w.Header() 37 | for k, v := range o.Meta { 38 | h.Set(k, v) 39 | } 40 | h.Set("allow", allowMethods) 41 | if o := corsMatch(&s.CORS, r.Header.Get("origin")); o != "" { 42 | h.Set("access-control-allow-origin", o) 43 | if r.Method == "OPTIONS" { 44 | h.Set("access-control-allow-methods", allowMethods) 45 | h.Set("access-control-allow-headers", r.Header.Get("access-control-request-headers")) 46 | h.Set("access-control-expose-headers", "Location, Etag, Content-Disposition") 47 | if s.CORS.MaxAge != "" { 48 | h.Set("access-control-max-age", s.CORS.MaxAge) 49 | } 50 | } 51 | } 52 | 53 | // redirect 54 | if v := o.Redirect(); v != "" && r.Method != "OPTIONS" { 55 | h.Set("location", v) 56 | w.WriteHeader(o.RedirectCode()) 57 | return nil 58 | } 59 | 60 | // body 61 | if r.Method == "GET" { 62 | _, err := io.Copy(w, o.Body) 63 | return err 64 | } 65 | 66 | return nil 67 | } 68 | 69 | // HandleChangeHook handles Object Change Notifications as described at 70 | // https://cloud.google.com/storage/docs/object-change-notification. 71 | // It removes objects from cache. 72 | func (s *Storage) HandleChangeHook(w http.ResponseWriter, r *http.Request) { 73 | // skip sync requests 74 | if v := r.Header.Get("x-goog-resource-state"); v == "sync" { 75 | return 76 | } 77 | 78 | // this is not a client request, so don't use newContext. 79 | ctx := appengine.NewContext(r) 80 | // we only care about name and the bucket 81 | body := struct{ Name, Bucket string }{} 82 | if err := json.NewDecoder(r.Body).Decode(&body); err != nil { 83 | log.Errorf(ctx, "json.Decode: %v", err.Error()) 84 | return 85 | } 86 | if err := s.PurgeCache(ctx, body.Bucket, body.Name); err != nil { 87 | log.Errorf(ctx, "s.PurgeCache(%q, %q): %v", body.Bucket, body.Name, err) 88 | w.WriteHeader(http.StatusInternalServerError) // let GCS retry 89 | } 90 | } 91 | 92 | // ValidMethod reports whether m is a supported HTTP method. 93 | func ValidMethod(m string) bool { 94 | return strings.Index(allowMethods, m) >= 0 95 | } 96 | 97 | func corsMatch(cors *CORS, o string) string { 98 | if len(cors.Origin) == 0 { 99 | return "" 100 | } 101 | if cors.Origin[0] == "*" { 102 | return "*" 103 | } 104 | if cors.Origin[0] == o { 105 | return o 106 | } 107 | i := sort.SearchStrings(cors.Origin, o) 108 | if i < len(cors.Origin) && cors.Origin[i] == o { 109 | return o 110 | } 111 | return "" 112 | } 113 | -------------------------------------------------------------------------------- /handle_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package weasel 16 | 17 | import ( 18 | "io/ioutil" 19 | "net/http" 20 | "net/http/httptest" 21 | "strings" 22 | "testing" 23 | 24 | "google.golang.org/appengine" 25 | "google.golang.org/appengine/memcache" 26 | ) 27 | 28 | func TestServeRedirect(t *testing.T) { 29 | const redir = "https://example.com" 30 | stor := &Storage{} 31 | o := &Object{ 32 | Meta: map[string]string{ 33 | metaRedirect: redir, 34 | metaRedirectCode: "302", 35 | }, 36 | } 37 | r, _ := http.NewRequest("GET", "/", nil) 38 | w := httptest.NewRecorder() 39 | 40 | err := stor.ServeObject(w, r, o) 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | if w.Code != http.StatusFound { 45 | t.Errorf("w.Code = %d; want %d", w.Code, http.StatusFound) 46 | } 47 | if v := w.Header().Get("location"); v != redir { 48 | t.Errorf("location = %q; want %q", v, redir) 49 | } 50 | } 51 | 52 | func TestServeCross(t *testing.T) { 53 | stor := &Storage{ 54 | CORS: CORS{ 55 | Origin: []string{"*"}, 56 | MaxAge: "300", 57 | }, 58 | } 59 | o := &Object{Body: ioutil.NopCloser(strings.NewReader("test"))} 60 | r, _ := http.NewRequest("GET", "/", nil) 61 | r.Header.Set("origin", "https://example.com") 62 | w := httptest.NewRecorder() 63 | 64 | err := stor.ServeObject(w, r, o) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | if w.Code != http.StatusOK { 69 | t.Errorf("w.Code = %d; want %d", w.Code, http.StatusOK) 70 | } 71 | if v := w.Body.String(); v != "test" { 72 | t.Errorf("didn't want w.Body: %q", v) 73 | } 74 | if v := w.Header().Get("access-control-allow-origin"); v != "*" { 75 | t.Errorf("allow-origin: %q; want *", v) 76 | } 77 | } 78 | 79 | func TestServeCrossPreflight(t *testing.T) { 80 | stor := &Storage{ 81 | CORS: CORS{ 82 | Origin: []string{"*"}, 83 | MaxAge: "300", 84 | }, 85 | } 86 | o := &Object{ 87 | Meta: map[string]string{ 88 | metaRedirect: "https://example.org", 89 | metaRedirectCode: "302", 90 | }, 91 | Body: ioutil.NopCloser(strings.NewReader("test")), 92 | } 93 | r, _ := http.NewRequest("OPTIONS", "/", nil) 94 | r.Header.Set("origin", "https://example.com") 95 | r.Header.Set("access-control-request-method", "GET") 96 | r.Header.Set("access-control-request-headers", "X-Foo") 97 | w := httptest.NewRecorder() 98 | 99 | err := stor.ServeObject(w, r, o) 100 | if err != nil { 101 | t.Fatal(err) 102 | } 103 | if w.Code != http.StatusOK { 104 | t.Errorf("w.Code = %d; want %d", w.Code, http.StatusOK) 105 | } 106 | if v := w.Body.String(); v != "" { 107 | t.Errorf("didn't want w.Body: %q", v) 108 | } 109 | if v := w.Header().Get("access-control-allow-origin"); v != "*" { 110 | t.Errorf("allow-origin: %q; want *", v) 111 | } 112 | if v := w.Header().Get("access-control-allow-headers"); v != "X-Foo" { 113 | t.Errorf("allow-headers: %q; want X-Foo", v) 114 | } 115 | if v := w.Header().Get("access-control-max-age"); v != "300" { 116 | t.Errorf("max-age: %q; want 300", v) 117 | } 118 | want := "GET, HEAD, OPTIONS" 119 | if v := w.Header().Get("access-control-allow-methods"); v != want { 120 | t.Errorf("allow-methods: %q; want %q", v, want) 121 | } 122 | } 123 | 124 | func TestHook(t *testing.T) { 125 | var stor Storage 126 | r, _ := testInstance.NewRequest("GET", "/", nil) 127 | ctx := appengine.NewContext(r) 128 | cacheKey := stor.CacheKey("dummy", "path/obj") 129 | item := &memcache.Item{Key: cacheKey, Value: []byte("ignored")} 130 | if err := memcache.Set(ctx, item); err != nil { 131 | t.Fatal(err) 132 | } 133 | 134 | body := `{"bucket": "dummy", "name": "path/obj"}` 135 | req, _ := testInstance.NewRequest("POST", "/hook", strings.NewReader(body)) 136 | res := httptest.NewRecorder() 137 | stor.HandleChangeHook(res, req) 138 | if res.Code != http.StatusOK { 139 | t.Errorf("res.Code = %d; want %d", res.Code, http.StatusOK) 140 | } 141 | // Must remove cached item. 142 | if _, err := memcache.Get(ctx, cacheKey); err != memcache.ErrCacheMiss { 143 | t.Fatalf("memcache.Get(%q): %v; want ErrCacheMiss", cacheKey, err) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /internal/token.go: -------------------------------------------------------------------------------- 1 | // Package internal contains utilities internal to the weasel packages. 2 | package internal 3 | 4 | import ( 5 | "context" 6 | "golang.org/x/oauth2" 7 | "golang.org/x/oauth2/google" 8 | ) 9 | 10 | // AETokenSource returns App Engine OAuth2 token source 11 | // given a context.Context and a slice of scopes. 12 | // It is a stubbed static token source during testing. 13 | var AETokenSource = func(ctx context.Context, scope ...string) oauth2.TokenSource { 14 | ts, err := google.DefaultTokenSource(ctx, scope...) 15 | if (err != nil) { 16 | panic(err) 17 | } 18 | return ts 19 | } 20 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package weasel 16 | 17 | import ( 18 | "context" 19 | "flag" 20 | "log" 21 | "os" 22 | "strings" 23 | "testing" 24 | 25 | "github.com/google/weasel/internal" 26 | 27 | "golang.org/x/oauth2" 28 | "google.golang.org/appengine/aetest" 29 | ) 30 | 31 | // global App Engine test instance, initialized and shutdown in TestMain. 32 | var testInstance aetest.Instance 33 | 34 | func TestMain(m *testing.M) { 35 | flag.Parse() 36 | 37 | var err error 38 | testInstance, err = aetest.NewInstance(nil) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | // app engine token source stub 44 | internal.AETokenSource = func(c context.Context, scopes ...string) oauth2.TokenSource { 45 | t := &oauth2.Token{ 46 | AccessToken: "InvalidToken:" + strings.Join(scopes, ","), 47 | } 48 | return oauth2.StaticTokenSource(t) 49 | } 50 | 51 | code := m.Run() 52 | testInstance.Close() 53 | os.Exit(code) 54 | } 55 | -------------------------------------------------------------------------------- /object.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package weasel 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "io" 21 | "net/http" 22 | "strconv" 23 | "time" 24 | 25 | "google.golang.org/appengine/log" 26 | "google.golang.org/appengine/memcache" 27 | ) 28 | 29 | const ( 30 | // object custom metadata 31 | metaRedirect = "x-goog-meta-redirect" 32 | metaRedirectCode = "x-goog-meta-redirect-code" 33 | 34 | // memcache settings 35 | cacheItemMax = 1 << 20 // max size per item, in bytes 36 | cacheItemExpiry = 24 * time.Hour 37 | ) 38 | 39 | // objectHeaders is a slice of headers propagated from a GCS object. 40 | var objectHeaders = []string{ 41 | "cache-control", 42 | "content-disposition", 43 | "content-type", 44 | "etag", 45 | "last-modified", 46 | metaRedirect, 47 | metaRedirectCode, 48 | } 49 | 50 | // Object represents a single GCS object. 51 | type Object struct { 52 | Meta map[string]string 53 | Body io.ReadCloser 54 | } 55 | 56 | // Redirect returns o's redirect URL, zero string otherwise. 57 | func (o *Object) Redirect() string { 58 | return o.Meta[metaRedirect] 59 | } 60 | 61 | // RedirectCode returns o's HTTP response code for redirect. 62 | // It defaults to http.StatusMovedPermanently. 63 | func (o *Object) RedirectCode() int { 64 | c, err := strconv.Atoi(o.Meta[metaRedirectCode]) 65 | if err != nil { 66 | c = http.StatusMovedPermanently 67 | } 68 | return c 69 | } 70 | 71 | // objectBuf implements io.ReadCloser for Object.Body. 72 | // It stores all r.Read results in its buf and caches exported fields 73 | // in memcache when Read returns io.EOF. 74 | type objectBuf struct { 75 | Meta map[string]string 76 | Body []byte // set after rc returns io.EOF 77 | 78 | r io.Reader 79 | buf bytes.Buffer 80 | key string // cache key 81 | ctx context.Context // memcache context 82 | } 83 | 84 | func (b *objectBuf) Read(p []byte) (int, error) { 85 | n, err := b.r.Read(p) 86 | if n > 0 && b.buf.Len() < cacheItemMax { 87 | b.buf.Write(p[:n]) 88 | } 89 | if err == io.EOF && b.buf.Len() < cacheItemMax { 90 | b.Body = b.buf.Bytes() 91 | item := memcache.Item{ 92 | Key: b.key, 93 | Object: b, 94 | Expiration: cacheItemExpiry, 95 | } 96 | if err := memcache.Gob.Set(b.ctx, &item); err != nil { 97 | log.Errorf(b.ctx, "memcache.Gob.Set(%q): %v", b.key, err) 98 | } 99 | } 100 | return n, err 101 | } 102 | 103 | func (b *objectBuf) Close() error { 104 | if c, ok := b.r.(io.Closer); ok { 105 | return c.Close() 106 | } 107 | return nil 108 | } 109 | -------------------------------------------------------------------------------- /object_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package weasel 16 | 17 | import ( 18 | "net/http" 19 | "testing" 20 | ) 21 | 22 | func TestObjectRedirect(t *testing.T) { 23 | o := &Object{Meta: map[string]string{metaRedirect: "/new/path"}} 24 | if v := o.Redirect(); v != "/new/path" { 25 | t.Errorf("o.Redirect() = %q; want /new/path", v) 26 | } 27 | if v := o.RedirectCode(); v != http.StatusMovedPermanently { 28 | t.Errorf("o.RedirectCode() = %d; want %d", v, http.StatusMovedPermanently) 29 | } 30 | 31 | o = &Object{Meta: map[string]string{metaRedirectCode: "333"}} 32 | if v := o.RedirectCode(); v != 333 { 33 | t.Errorf("o.RedirectCode() = %d; want 333", v) 34 | } 35 | 36 | o = &Object{Meta: map[string]string{metaRedirectCode: "invalid"}} 37 | if v := o.RedirectCode(); v != http.StatusMovedPermanently { 38 | t.Errorf("o.RedirectCode() = %d; want %d", v, http.StatusMovedPermanently) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /server/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "context" 19 | "flag" 20 | "log" 21 | "os" 22 | "strings" 23 | "testing" 24 | 25 | "github.com/google/weasel/internal" 26 | 27 | "golang.org/x/oauth2" 28 | "google.golang.org/appengine/aetest" 29 | ) 30 | 31 | // global App Engine test instance, initialized and shutdown in TestMain. 32 | var testInstance aetest.Instance 33 | 34 | func TestMain(m *testing.M) { 35 | flag.Parse() 36 | 37 | var err error 38 | testInstance, err = aetest.NewInstance(nil) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | // app engine token source stub 44 | internal.AETokenSource = func(c context.Context, scopes ...string) oauth2.TokenSource { 45 | t := &oauth2.Token{ 46 | AccessToken: "InvalidToken:" + strings.Join(scopes, ","), 47 | } 48 | return oauth2.StaticTokenSource(t) 49 | } 50 | 51 | code := m.Run() 52 | testInstance.Close() 53 | os.Exit(code) 54 | } 55 | -------------------------------------------------------------------------------- /server/server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package server provides a simple frontend in form of an App Engine app 16 | // built atop the weasel.Storage. 17 | // See README.md for the design details. 18 | // 19 | // This package is a work in progress and makes no API stability promises. 20 | // 21 | // An exaple usage for App Engine Standard: 22 | // 23 | // # app.yaml 24 | // runtime: go 25 | // api_version: go1 26 | // handlers: 27 | // - url: /.* 28 | // script: _go_app 29 | // 30 | // # app.go 31 | // package app 32 | // 33 | // import ( 34 | // "github.com/google/weasel" 35 | // "github.com/google/weasel/server" 36 | // ) 37 | // 38 | // func init() { 39 | // conf := &server.Config{ 40 | // Storage: weasel.DefaultStorage, 41 | // Buckets: map[string]string{ 42 | // "default": "my-gcs-bucket", 43 | // }, 44 | // HookPath: "/-/flush-gcs-cache", 45 | // } 46 | // server.Init(nil, conf) 47 | // } 48 | // 49 | // The "/-/flush-gcs-cache" needs to be hooked up with "my-gcs-bucket" manually 50 | // using Object Change Notifications. See the following page for more details: 51 | // https://cloud.google.com/storage/docs/object-change-notification 52 | package server 53 | 54 | import ( 55 | "context" 56 | "net/http" 57 | "time" 58 | 59 | "github.com/google/weasel" 60 | 61 | "google.golang.org/appengine" 62 | "google.golang.org/appengine/log" 63 | ) 64 | 65 | // Used to set STS header value when serving over TLS. 66 | const stsValue = "max-age=10886400; includeSubDomains; preload" 67 | 68 | // Init registers server handlers on the provided mux. 69 | // If the mux argument is nil, http.DefaultServeMux is used. 70 | // 71 | // See package doc for a usage example. 72 | func Init(mux *http.ServeMux, conf *Config) { 73 | if mux == nil { 74 | mux = http.DefaultServeMux 75 | } 76 | for host, redir := range conf.Redirects { 77 | mux.Handle(host, redirectHandler(redir, http.StatusMovedPermanently)) 78 | } 79 | s := &server{ 80 | storage: conf.Storage, 81 | buckets: conf.Buckets, 82 | tlsOnly: make(map[string]struct{}, len(conf.TLSOnly)), 83 | } 84 | for _, h := range conf.TLSOnly { 85 | s.tlsOnly[h] = struct{}{} 86 | } 87 | mux.Handle(conf.webroot(), s) 88 | if conf.HookPath != "" { 89 | mux.HandleFunc(conf.HookPath, conf.Storage.HandleChangeHook) 90 | } 91 | } 92 | 93 | // Config is used to init the server. 94 | // See Init for more details. 95 | type Config struct { 96 | // Storage provides server with the content access. 97 | Storage *weasel.Storage 98 | 99 | // Buckets defines a mapping between hosts 100 | // and GCS buckets the responses should be served from. 101 | // The map must contain at least "default" key. 102 | Buckets map[string]string 103 | 104 | // WebRoot is the content serving root pattern. 105 | // If empty, default is used. 106 | // Default value is "/". 107 | WebRoot string 108 | 109 | // GCS object change notification hook pattern. 110 | // If empty, no hook handler will be setup during Init. 111 | HookPath string 112 | 113 | // Redirects is a map of URLs the app will permanently redirect to 114 | // when the request host and path match a key. 115 | // Map values must not end with "/" and cannot contain query string. 116 | Redirects map[string]string 117 | 118 | // TLSOnly forces TLS connection for the specified host names. 119 | TLSOnly []string 120 | } 121 | 122 | func (c *Config) webroot() string { 123 | if c.WebRoot != "" { 124 | return c.WebRoot 125 | } 126 | return "/" 127 | } 128 | 129 | type server struct { 130 | // The GCS storage to serve content from. 131 | storage *weasel.Storage 132 | 133 | // Contains hostnames forced to be server over TLS. 134 | tlsOnly map[string]struct{} 135 | 136 | // Defines a mapping between hosts 137 | // and GCS buckets the responses should be served from. 138 | // The map must contain at least "default" key. 139 | buckets map[string]string 140 | } 141 | 142 | // ServeHTTP responds with a GCS object contents, preserving its original headers 143 | // listed in objectHeaders. 144 | // The bucket is identifed by matching r.Host against config.Buckets map keys. 145 | // Default bucket is used if no match is found. 146 | // 147 | // Only GET, HEAD and OPTIONS methods are allowed. 148 | func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) { 149 | _, forceTLS := s.tlsOnly[r.Host] 150 | if forceTLS && r.Header.Get("X-Forwarded-Proto") == "https" { 151 | w.Header().Set("Strict-Transport-Security", stsValue) 152 | } 153 | if !weasel.ValidMethod(r.Method) { 154 | http.Error(w, "", http.StatusMethodNotAllowed) 155 | return 156 | } 157 | if forceTLS && r.Header.Get("X-Forwarded-Proto") == "http" { 158 | u := "https://" + r.Host + r.URL.Path 159 | if r.URL.RawQuery != "" { 160 | u += "?" + r.URL.RawQuery 161 | } 162 | http.Redirect(w, r, u, http.StatusMovedPermanently) 163 | return 164 | } 165 | 166 | ctx, cancel := context.WithTimeout(appengine.NewContext(r), 10*time.Second) 167 | defer cancel() 168 | bucket := s.bucketForHost(r.Host) 169 | oname := r.URL.Path[1:] 170 | 171 | o, err := s.storage.OpenFile(ctx, bucket, oname) 172 | if err != nil { 173 | code := http.StatusInternalServerError 174 | if errf, ok := err.(*weasel.FetchError); ok { 175 | code = errf.Code 176 | } 177 | serveError(w, code, "") 178 | if code != http.StatusNotFound { 179 | log.Errorf(ctx, "%s/%s: %v", bucket, oname, err) 180 | } 181 | return 182 | } 183 | if err := s.storage.ServeObject(w, r, o); err != nil { 184 | log.Errorf(ctx, "%s/%s: %v", bucket, oname, err) 185 | } 186 | o.Body.Close() 187 | } 188 | 189 | // bucketForHost returns a bucket name mapped to the host. 190 | // Default bucket name is return if no match found. 191 | func (s *server) bucketForHost(host string) string { 192 | if b, ok := s.buckets[host]; ok { 193 | return b 194 | } 195 | return s.buckets["default"] 196 | } 197 | 198 | // redirectHandler creates a new handler which redirects all requests 199 | // to the specified url, preserving original path and raw query. 200 | func redirectHandler(url string, code int) http.Handler { 201 | // TODO: parse url and support path, query, etc. 202 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 203 | u := url + r.URL.Path 204 | if r.URL.RawQuery != "" { 205 | u += "?" + r.URL.RawQuery 206 | } 207 | http.Redirect(w, r, u, code) 208 | }) 209 | } 210 | 211 | func serveError(w http.ResponseWriter, code int, msg string) { 212 | if msg == "" { 213 | msg = http.StatusText(code) 214 | } 215 | w.WriteHeader(code) 216 | // TODO: render some template 217 | w.Write([]byte(msg)) 218 | } 219 | -------------------------------------------------------------------------------- /server/server_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package server 16 | 17 | import ( 18 | "net/http" 19 | "net/http/httptest" 20 | "strings" 21 | "testing" 22 | 23 | "github.com/google/weasel" 24 | 25 | "google.golang.org/appengine" 26 | "google.golang.org/appengine/memcache" 27 | ) 28 | 29 | func TestInit(t *testing.T) { 30 | Init(nil, &Config{ 31 | WebRoot: "/root/", 32 | HookPath: "/flush-cache", 33 | Redirects: map[string]string{"example.org/": "redir.host"}, 34 | TLSOnly: []string{"tls.example.org"}, 35 | }) 36 | patterns := []struct{ in, out string }{ 37 | {"/", ""}, 38 | {"/root", "/root/"}, 39 | {"/root/", "/root/"}, 40 | {"/root/foo", "/root/"}, 41 | {"/flush-cache", "/flush-cache"}, 42 | {"http://example.org/", "example.org/"}, 43 | } 44 | for i, p := range patterns { 45 | r, err := http.NewRequest("GET", p.in, nil) 46 | if err != nil { 47 | t.Errorf("%d: NewRequest(%q): %v", i, p.in, err) 48 | continue 49 | } 50 | if _, v := http.DefaultServeMux.Handler(r); v != p.out { 51 | t.Errorf("%d: Handler(%q) = %q; want %q", i, p.in, v, p.out) 52 | } 53 | } 54 | r, _ := testInstance.NewRequest("GET", "http://tls.example.org/root/", nil) 55 | r.Header.Set("X-Forwarded-Proto", "http") 56 | w := httptest.NewRecorder() 57 | http.DefaultServeMux.ServeHTTP(w, r) 58 | if w.Code != http.StatusMovedPermanently { 59 | t.Errorf("w.Code = %d; want %d", w.Code, http.StatusMovedPermanently) 60 | } 61 | } 62 | 63 | func TestRedirect(t *testing.T) { 64 | const ( 65 | redirectTo = "https://www.example.com" 66 | code = http.StatusFound 67 | ) 68 | handler := redirectHandler(redirectTo, code) 69 | urls := []string{"/", "/page", "/page/", "/page?with=query"} 70 | for _, u := range urls { 71 | req, err := testInstance.NewRequest("GET", u, nil) 72 | if err != nil { 73 | t.Errorf("%s: %v", u, err) 74 | continue 75 | } 76 | req.Host = "example.org" 77 | res := httptest.NewRecorder() 78 | handler.ServeHTTP(res, req) 79 | if res.Code != code { 80 | t.Errorf("%s: res.Code = %d; want %d", u, res.Code, code) 81 | } 82 | redir := redirectTo + u 83 | if v := res.Header().Get("location"); v != redir { 84 | t.Errorf("%s: location = %q; want %q", u, v, redir) 85 | } 86 | } 87 | } 88 | 89 | func TestTLSOnly(t *testing.T) { 90 | gcs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 91 | // empty 200 OK response 92 | })) 93 | defer gcs.Close() 94 | srv := &server{ 95 | storage: &weasel.Storage{Base: gcs.URL}, 96 | tlsOnly: map[string]struct{}{"example.com": {}}, 97 | } 98 | r, _ := testInstance.NewRequest("GET", "http://example.com/page?foo=bar", nil) 99 | r.Header.Set("X-Forwarded-Proto", "http") 100 | w := httptest.NewRecorder() 101 | srv.ServeHTTP(w, r) 102 | if w.Code != http.StatusMovedPermanently { 103 | t.Errorf("w.Code = %d; want %d", w.Code, http.StatusMovedPermanently) 104 | } 105 | want := "https://example.com/page?foo=bar" 106 | if l := w.Header().Get("location"); l != want { 107 | t.Errorf("location = %q; want %q", l, want) 108 | } 109 | 110 | r, _ = testInstance.NewRequest("GET", "https://example.com/page?foo=bar", nil) 111 | r.Header.Set("X-Forwarded-Proto", "https") 112 | w = httptest.NewRecorder() 113 | srv.ServeHTTP(w, r) 114 | if v := w.Header().Get("strict-transport-security"); v != stsValue { 115 | t.Errorf("strict-transport-security: %q; want %q", v, stsValue) 116 | } 117 | } 118 | 119 | func TestServe_DefaultGCS(t *testing.T) { 120 | const ( 121 | bucket = "default-bucket" 122 | reqFile = "/dir/" 123 | realFile = bucket + "/dir/index.html" 124 | contents = "contents" 125 | contentType = "text/plain" 126 | cacheControl = "public,max-age=0" 127 | // dev_appserver app identity stub 128 | authorization = "Bearer InvalidToken:https://www.googleapis.com/auth/devstorage.read_only" 129 | ) 130 | 131 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 132 | if r.URL.Path[1:] != realFile { 133 | t.Errorf("r.URL.Path = %q; want /%s", r.URL.Path, realFile) 134 | } 135 | if v := r.Header.Get("authorization"); !strings.HasPrefix(v, authorization) { 136 | t.Errorf("auth = %q; want prefix %q", v, authorization) 137 | } 138 | if v, exist := r.Header["X-Foo"]; exist { 139 | t.Errorf("found x-foo: %q", v) 140 | } 141 | // weasel client => GCS always uses gzip where available 142 | if v := r.Header.Get("accept-encoding"); v != "gzip" { 143 | t.Errorf("accept-encoding = %q; want 'gzip'", v) 144 | } 145 | w.Header().Set("cache-control", cacheControl) 146 | w.Header().Set("content-type", contentType) 147 | w.Header().Set("x-test", "should not propagate") 148 | w.Write([]byte(contents)) 149 | })) 150 | defer ts.Close() 151 | srv := &server{ 152 | storage: &weasel.Storage{ 153 | Base: ts.URL, 154 | Index: "index.html", 155 | }, 156 | buckets: map[string]string{"default": bucket}, 157 | } 158 | 159 | req, _ := testInstance.NewRequest("GET", reqFile, nil) 160 | req.Header.Set("accept-encoding", "client/accept") 161 | req.Header.Set("x-foo", "bar") 162 | // make sure we're not getting memcached results 163 | if err := memcache.Flush(appengine.NewContext(req)); err != nil { 164 | t.Fatal(err) 165 | } 166 | 167 | res := httptest.NewRecorder() 168 | srv.ServeHTTP(res, req) 169 | if res.Code != http.StatusOK { 170 | t.Errorf("res.Code = %d; want %d", res.Code, http.StatusOK) 171 | } 172 | if v := res.Header().Get("cache-control"); v != cacheControl { 173 | t.Errorf("cache-control = %q; want %q", v, cacheControl) 174 | } 175 | if v := res.Header().Get("content-type"); v != contentType { 176 | t.Errorf("content-type = %q; want %q", v, contentType) 177 | } 178 | if v := res.Header().Get("x-test"); v != "" { 179 | t.Errorf("found x-test header: %q", v) 180 | } 181 | if s := res.Body.String(); s != contents { 182 | t.Errorf("res.Body = %q; want %q", s, contents) 183 | } 184 | } 185 | 186 | func TestServe_Methods(t *testing.T) { 187 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 188 | w.Header().Set("content-type", "text/plain") 189 | w.Write([]byte("methods test")) 190 | })) 191 | defer ts.Close() 192 | srv := &server{ 193 | storage: &weasel.Storage{Base: ts.URL}, 194 | buckets: map[string]string{"default": "bucket"}, 195 | } 196 | 197 | tests := []struct { 198 | method, body string 199 | code int 200 | }{ 201 | {"HEAD", "", http.StatusOK}, 202 | {"OPTIONS", "", http.StatusOK}, 203 | // it is important that GET comes last to verify requests like HEAD 204 | // do not corrupt object cache 205 | {"GET", "methods test", http.StatusOK}, 206 | 207 | {"PUT", "", http.StatusMethodNotAllowed}, 208 | {"POST", "", http.StatusMethodNotAllowed}, 209 | {"DELETE", "", http.StatusMethodNotAllowed}, 210 | } 211 | for i, test := range tests { 212 | r, _ := testInstance.NewRequest(test.method, "/file.txt", nil) 213 | rw := httptest.NewRecorder() 214 | srv.ServeHTTP(rw, r) 215 | if rw.Code != test.code { 216 | t.Errorf("%d: rw.Code = %d; want %d", i, rw.Code, test.code) 217 | } 218 | if v := strings.TrimSpace(rw.Body.String()); v != test.body { 219 | t.Errorf("%d: rw.Body = %q; want %q", i, v, test.body) 220 | } 221 | } 222 | } 223 | 224 | func TestServe_GCSErrors(t *testing.T) { 225 | const code = http.StatusBadRequest 226 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 227 | w.WriteHeader(code) 228 | })) 229 | defer ts.Close() 230 | srv := &server{ 231 | storage: &weasel.Storage{Base: ts.URL}, 232 | buckets: map[string]string{"default": "bucket"}, 233 | } 234 | 235 | req, err := testInstance.NewRequest("GET", "/bad", nil) 236 | if err != nil { 237 | t.Fatal(err) 238 | } 239 | res := httptest.NewRecorder() 240 | srv.ServeHTTP(res, req) 241 | if res.Code != code { 242 | t.Errorf("res.Code = %d; want %d", res.Code, code) 243 | } 244 | } 245 | 246 | func TestServe_NoTrailSlash(t *testing.T) { 247 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 248 | if r.URL.Path != "/bucket/dir-one/two/index.html" { 249 | w.WriteHeader(http.StatusNotFound) 250 | return 251 | } 252 | // stat request 253 | if r.Method != "HEAD" { 254 | t.Errorf("r.Method = %q; want HEAD", r.Method) 255 | } 256 | })) 257 | defer ts.Close() 258 | srv := &server{ 259 | storage: &weasel.Storage{ 260 | Base: ts.URL, 261 | Index: "index.html", 262 | }, 263 | buckets: map[string]string{"default": "bucket"}, 264 | } 265 | 266 | req, _ := testInstance.NewRequest("GET", "/dir-one/two", nil) 267 | // make sure we're not getting memcached results 268 | if err := memcache.Flush(appengine.NewContext(req)); err != nil { 269 | t.Fatal(err) 270 | } 271 | res := httptest.NewRecorder() 272 | srv.ServeHTTP(res, req) 273 | if res.Code != http.StatusMovedPermanently { 274 | t.Errorf("res.Code = %d; want %d", res.Code, http.StatusMovedPermanently) 275 | } 276 | loc := "/dir-one/two/" 277 | if v := res.Header().Get("location"); v != loc { 278 | t.Errorf("location = %q; want %q", v, loc) 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /storage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package weasel provides means for serving content from a Google Cloud Storage (GCS) 16 | // bucket, suitable for hosting on Google App Engine. 17 | // See README.md for the design details. 18 | // 19 | // This package is a work in progress and makes no API stability promises. 20 | package weasel 21 | 22 | import ( 23 | "bytes" 24 | "context" 25 | "fmt" 26 | "io/ioutil" 27 | "net/http" 28 | "path" 29 | "path/filepath" 30 | "strings" 31 | "time" 32 | 33 | "github.com/google/weasel/internal" 34 | 35 | "golang.org/x/oauth2" 36 | "google.golang.org/appengine/log" 37 | "google.golang.org/appengine/memcache" 38 | "google.golang.org/appengine/urlfetch" 39 | ) 40 | 41 | // Google Cloud Storage OAuth2 scopes. 42 | const scopeStorageRead = "https://www.googleapis.com/auth/devstorage.read_only" 43 | 44 | // DefaultStorage is a Storage with sensible default parameters. 45 | var DefaultStorage = &Storage{ 46 | Base: "https://storage.googleapis.com", 47 | Index: "index.html", 48 | CORS: CORS{ 49 | Origin: []string{"*"}, 50 | MaxAge: "86400", 51 | }, 52 | } 53 | 54 | // CORS is a Storage cross-origin settings. 55 | type CORS struct { 56 | Origin []string // allowed origins 57 | MaxAge string // preflight cache, in seconds 58 | } 59 | 60 | // Storage incapsulates configuration params for retrieveing and serving GCS objects. 61 | type Storage struct { 62 | Base string // GCS service base URL, e.g. "https://storage.googleapis.com". 63 | Index string // Appended to an object name in certain cases, e.g. "index.html". 64 | CORS CORS 65 | } 66 | 67 | // OpenFile abstracts Open and treats object name like a file path. 68 | func (s *Storage) OpenFile(ctx context.Context, bucket, name string) (*Object, error) { 69 | if name == "" || strings.HasSuffix(name, "/") { 70 | name += s.Index 71 | } 72 | 73 | // stat /dir/index.html if name is /dir, concurrently 74 | checkStat := !strings.HasSuffix(name, s.Index) && filepath.Ext(name) == "" 75 | type stat struct { 76 | o *Object 77 | err error 78 | } 79 | var ch chan *stat 80 | if checkStat { 81 | ch = make(chan *stat, 1) 82 | go func() { 83 | o, err := s.Stat(ctx, bucket, path.Join(name, s.Index)) 84 | ch <- &stat{o, err} 85 | close(ch) 86 | }() 87 | } 88 | 89 | // try the original object meanwhile 90 | o, err := s.Open(ctx, bucket, name) 91 | if err == nil || !checkStat { 92 | return o, err 93 | } 94 | // Return non-404 errors right away, even when checkStat == true. 95 | // Note that GCS now may respond with 403 Forbidden 96 | // for nonexistent objects. 97 | if ferr, ok := err.(*FetchError); ok && ferr.Code != 404 && ferr.Code != 403 { 98 | return nil, err 99 | } 100 | 101 | // wait some time for stat obj 102 | // TODO: use ctxhttp 103 | select { 104 | case <-time.After(5 * time.Second): 105 | log.Errorf(ctx, "s.Stat(bucket=%q) timeout", bucket) 106 | // return original Open error 107 | return nil, err 108 | case res := <-ch: 109 | if res.err != nil { 110 | // return original Open error 111 | return nil, err 112 | } 113 | o = res.o 114 | } 115 | if o.Redirect() == "" { 116 | o = &Object{ 117 | Body: ioutil.NopCloser(bytes.NewReader(nil)), 118 | Meta: map[string]string{ 119 | metaRedirect: path.Join("/", name) + "/", 120 | }, 121 | } 122 | } 123 | return o, nil 124 | } 125 | 126 | // Open retrieves GCS object name of the bucket from cache or network. 127 | // Objects fetched from the network are cached before returning 128 | // from this function. 129 | func (s *Storage) Open(ctx context.Context, bucket, name string) (*Object, error) { 130 | key := s.CacheKey(bucket, name) 131 | o, err := getCache(ctx, key) 132 | if err != nil { 133 | u := fmt.Sprintf("%s/%s", s.Base, path.Join(bucket, name)) 134 | o, err = fetch(ctx, u, key) 135 | } 136 | return o, err 137 | } 138 | 139 | // Stat is similar to Read except the returned Object.Body may be nil. 140 | // In the case where Body is not nil, calling Body.Close() is not required. 141 | func (s *Storage) Stat(ctx context.Context, bucket, name string) (*Object, error) { 142 | if o, err := getCache(ctx, s.CacheKey(bucket, name)); err == nil { 143 | return o, nil 144 | } 145 | u := fmt.Sprintf("%s/%s", s.Base, path.Join(bucket, name)) 146 | req, err := http.NewRequest("HEAD", u, nil) 147 | if err != nil { 148 | return nil, err 149 | } 150 | res, err := httpClient(ctx, scopeStorageRead).Do(req) 151 | if err != nil { 152 | return nil, err 153 | } 154 | defer res.Body.Close() 155 | if res.StatusCode != http.StatusOK { 156 | b, _ := ioutil.ReadAll(res.Body) 157 | return nil, &FetchError{ 158 | Msg: fmt.Sprintf("%s: %s", res.Status, b), 159 | Code: res.StatusCode, 160 | } 161 | } 162 | meta := make(map[string]string) 163 | for _, k := range objectHeaders { 164 | if v := res.Header.Get(k); v != "" { 165 | meta[k] = v 166 | } 167 | } 168 | return &Object{Meta: meta}, nil 169 | } 170 | 171 | // PurgeCache removes cached object from memcache. 172 | // It does not return an error in the case of cache miss. 173 | func (s *Storage) PurgeCache(ctx context.Context, bucket, name string) error { 174 | return purgeCache(ctx, s.CacheKey(bucket, name)) 175 | } 176 | 177 | // CacheKey returns a key to cache an object under, computed from 178 | // s.Base, bucket and then name. 179 | func (s *Storage) CacheKey(bucket, name string) string { 180 | return fmt.Sprintf("%s/%s", s.Base, path.Join(bucket, name)) 181 | } 182 | 183 | // fetch retrieves object from the given url. 184 | // The returned error will be of type FetchError if the storage responds 185 | // with an error code. 186 | // 187 | // The returned Object.Body will auto-cache in memcache if cacheKey 188 | // is provided and body length is within allowed cache limits. 189 | func fetch(ctx context.Context, url, cacheKey string) (*Object, error) { 190 | req, err := http.NewRequest("GET", url, nil) 191 | if err != nil { 192 | return nil, err 193 | } 194 | res, err := httpClient(ctx, scopeStorageRead).Do(req) 195 | if err != nil { 196 | return nil, err 197 | } 198 | if res.StatusCode > 399 { 199 | // FetchError takes precedence over i/o errors 200 | b, _ := ioutil.ReadAll(res.Body) 201 | res.Body.Close() 202 | return nil, &FetchError{ 203 | Msg: fmt.Sprintf("%s: %s", res.Status, b), 204 | Code: res.StatusCode, 205 | } 206 | } 207 | m := make(map[string]string) 208 | for _, k := range objectHeaders { 209 | if v := res.Header.Get(k); v != "" { 210 | m[k] = v 211 | } 212 | } 213 | rc := res.Body 214 | if cacheKey != "" && res.ContentLength < cacheItemMax { 215 | rc = &objectBuf{ 216 | Meta: m, 217 | r: res.Body, 218 | key: cacheKey, 219 | ctx: ctx, 220 | } 221 | } 222 | o := &Object{ 223 | Meta: m, 224 | Body: rc, 225 | } 226 | return o, nil 227 | } 228 | 229 | func getCache(ctx context.Context, key string) (*Object, error) { 230 | var b objectBuf 231 | if _, err := memcache.Gob.Get(ctx, key, &b); err != nil { 232 | if err != memcache.ErrCacheMiss { 233 | log.Errorf(ctx, "memcache.Gob.Get(%q): %v", key, err) 234 | } 235 | return nil, err 236 | } 237 | o := &Object{ 238 | Meta: b.Meta, 239 | Body: ioutil.NopCloser(bytes.NewReader(b.Body)), 240 | } 241 | return o, nil 242 | } 243 | 244 | func purgeCache(ctx context.Context, key string) error { 245 | err := memcache.Delete(ctx, key) 246 | if err == memcache.ErrCacheMiss { 247 | err = nil 248 | } 249 | return err 250 | } 251 | 252 | func httpClient(ctx context.Context, scopes ...string) *http.Client { 253 | t := &oauth2.Transport{ 254 | Source: internal.AETokenSource(ctx, scopes...), 255 | Base: &urlfetch.Transport{Context: ctx}, 256 | } 257 | return &http.Client{Transport: t} 258 | } 259 | 260 | // FetchError contains error code and message from a GCS response. 261 | type FetchError struct { 262 | Msg string 263 | Code int 264 | } 265 | 266 | // Error returns formatted FetchError. 267 | func (e *FetchError) Error() string { 268 | return fmt.Sprintf("FetchError %d: %s", e.Code, e.Msg) 269 | } 270 | -------------------------------------------------------------------------------- /storage_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 Google Inc. All Rights Reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package weasel 16 | 17 | import ( 18 | "io/ioutil" 19 | "net/http" 20 | "net/http/httptest" 21 | "reflect" 22 | "strings" 23 | "testing" 24 | 25 | "google.golang.org/appengine" 26 | "google.golang.org/appengine/memcache" 27 | ) 28 | 29 | func TestOpenFileIndex(t *testing.T) { 30 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 31 | // dev_appserver app identity stub 32 | auth := "Bearer InvalidToken:https://www.googleapis.com/auth/devstorage.read_only" 33 | if v := r.Header.Get("authorization"); !strings.HasPrefix(v, auth) { 34 | t.Errorf("auth = %q; want prefix %q", v, auth) 35 | } 36 | if r.URL.Path != "/bucket/dir/index" { 37 | t.Errorf("r.URL.Path = %q; want /bucket/dir/index", r.URL.Path) 38 | } 39 | // weasel client => GCS always uses gzip where available 40 | if v := r.Header.Get("accept-encoding"); v != "gzip" { 41 | t.Errorf("accept-encoding = %q; want 'gzip'", v) 42 | } 43 | w.Header().Set("content-type", "text/plain") 44 | w.Write([]byte("test file")) 45 | })) 46 | defer ts.Close() 47 | 48 | req, _ := testInstance.NewRequest("GET", "/", nil) 49 | ctx := appengine.NewContext(req) 50 | // make sure we're not getting memcached results 51 | if err := memcache.Flush(ctx); err != nil { 52 | t.Fatal(err) 53 | } 54 | 55 | stor := &Storage{Base: ts.URL, Index: "index"} 56 | obj, err := stor.OpenFile(ctx, "bucket", "/dir/") 57 | if err != nil { 58 | t.Fatalf("stor.OpenFile: %v", err) 59 | } 60 | defer obj.Body.Close() 61 | b, _ := ioutil.ReadAll(obj.Body) 62 | if string(b) != "test file" { 63 | t.Errorf("obj.Body = %q; want 'test file'", b) 64 | } 65 | } 66 | 67 | func TestOpenFileNoTrailSlash_404(t *testing.T) { 68 | testOpenFileNoTrailSlash(t, http.StatusNotFound) 69 | } 70 | 71 | func TestOpenFileNoTrailSlash_403(t *testing.T) { 72 | testOpenFileNoTrailSlash(t, http.StatusForbidden) 73 | } 74 | 75 | func testOpenFileNoTrailSlash(t *testing.T, status int) { 76 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 77 | if r.URL.Path != "/bucket/no/slash/index.html" { 78 | w.WriteHeader(status) 79 | return 80 | } 81 | // stat request 82 | if r.Method != "HEAD" { 83 | t.Errorf("r.Method = %q; want HEAD", r.Method) 84 | } 85 | })) 86 | defer ts.Close() 87 | 88 | r, _ := testInstance.NewRequest("GET", "/", nil) 89 | ctx := appengine.NewContext(r) 90 | // make sure we're not getting memcached results 91 | if err := memcache.Flush(ctx); err != nil { 92 | t.Fatal(err) 93 | } 94 | 95 | stor := &Storage{Base: ts.URL, Index: "index.html"} 96 | o, err := stor.OpenFile(ctx, "bucket", "/no/slash") 97 | if err != nil { 98 | t.Fatalf("stor.OpenFile: %v", err) 99 | } 100 | defer o.Body.Close() 101 | loc := "/no/slash/" 102 | if v := o.Redirect(); v != loc { 103 | t.Errorf("o.Redirect() = %q; want %q", v, loc) 104 | } 105 | if v := o.RedirectCode(); v != http.StatusMovedPermanently { 106 | t.Errorf("o.RedirectCode() = %d; want %d", v, http.StatusMovedPermanently) 107 | } 108 | } 109 | 110 | func TestOpenAndCache(t *testing.T) { 111 | const body = `{"foo":"bar"}` 112 | meta := map[string]string{"content-type": "application/json"} 113 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 114 | for k, v := range meta { 115 | w.Header().Set(k, v) 116 | } 117 | w.Write([]byte(body)) 118 | })) 119 | defer ts.Close() 120 | 121 | r, _ := testInstance.NewRequest("GET", "/", nil) 122 | ctx := appengine.NewContext(r) 123 | // make sure we're not getting memcached results 124 | if err := memcache.Flush(ctx); err != nil { 125 | t.Fatal(err) 126 | } 127 | 128 | stor := &Storage{Base: ts.URL} 129 | o, err := stor.Open(ctx, "bucket", "/file.json") 130 | if err != nil { 131 | t.Fatalf("stor.Open: %v", err) 132 | } 133 | defer o.Body.Close() 134 | b, err := ioutil.ReadAll(o.Body) 135 | if err != nil { 136 | t.Fatalf("ReadAll(o.Body): %v", err) 137 | } 138 | if string(b) != body { 139 | t.Errorf("o.Body = %q; want %q", b, body) 140 | } 141 | if !reflect.DeepEqual(o.Meta, meta) { 142 | t.Errorf("o.Meta = %+v; want %+v", o.Meta, meta) 143 | } 144 | 145 | key := stor.CacheKey("bucket", "/file.json") 146 | var ob objectBuf 147 | if _, err := memcache.Gob.Get(ctx, key, &ob); err != nil { 148 | t.Fatalf("memcache.Gob.Get(%q): %v", key, err) 149 | } 150 | if string(ob.Body) != body { 151 | t.Errorf("ob.Body = %q; want %q", ob.Body, body) 152 | } 153 | if !reflect.DeepEqual(ob.Meta, meta) { 154 | t.Errorf("ob.Meta = %+v; want %+v", ob.Meta, meta) 155 | } 156 | } 157 | 158 | func TestOpenFromCache(t *testing.T) { 159 | r, _ := testInstance.NewRequest("GET", "/", nil) 160 | ctx := appengine.NewContext(r) 161 | stor := &Storage{Base: "invalid"} // make sure we don't hit real GCS 162 | ob := &objectBuf{ 163 | Meta: map[string]string{ 164 | "content-type": "text/html", 165 | "cache-control": "public,max-age=10", 166 | }, 167 | Body: []byte("cached file"), 168 | } 169 | item := memcache.Item{ 170 | Key: stor.CacheKey("bucket", "TestOpenFromCache"), 171 | Object: ob, 172 | } 173 | if err := memcache.Gob.Set(ctx, &item); err != nil { 174 | t.Fatal(err) 175 | } 176 | 177 | o, err := stor.Open(ctx, "bucket", "TestOpenFromCache") 178 | if err != nil { 179 | t.Fatalf("stor.Open: %v", err) 180 | } 181 | defer o.Body.Close() 182 | if _, isbuf := o.Body.(*objectBuf); isbuf { 183 | t.Errorf("o.Body is *objectBuf") 184 | } 185 | if !reflect.DeepEqual(o.Meta, ob.Meta) { 186 | t.Errorf("o.Meta = %+v; want %+v", o.Meta, ob.Meta) 187 | } 188 | b, _ := ioutil.ReadAll(o.Body) 189 | if string(b) != "cached file" { 190 | t.Errorf("o.Body = %q; want 'cached file'", b) 191 | } 192 | } 193 | 194 | func TestOpenErr(t *testing.T) { 195 | ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 196 | w.WriteHeader(http.StatusBadRequest) 197 | })) 198 | defer ts.Close() 199 | 200 | req, _ := testInstance.NewRequest("GET", "/", nil) 201 | ctx := appengine.NewContext(req) 202 | stor := &Storage{Base: ts.URL} 203 | obj, err := stor.OpenFile(ctx, "bucket", "TestOpenErr") 204 | if err == nil { 205 | defer obj.Body.Close() 206 | t.Fatalf("stor.OpenFile: %+v; want error", obj) 207 | } 208 | errf, ok := err.(*FetchError) 209 | if !ok { 210 | t.Fatalf("want err to be a *FetchError") 211 | } 212 | if errf.Code != http.StatusBadRequest { 213 | t.Errorf("errf.Code = %d; want %d", errf.Code, http.StatusBadRequest) 214 | } 215 | } 216 | --------------------------------------------------------------------------------