├── .gitignore ├── .goreleaser.yaml ├── LICENSE ├── README.md ├── example.cfg ├── go.mod ├── go.sum ├── main.go └── model.go /.gitignore: -------------------------------------------------------------------------------- 1 | /ellycache 2 | 3 | dist/ 4 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | builds: 4 | - binary: ellycache 5 | targets: 6 | - darwin_amd64 7 | - darwin_arm64 8 | - linux_amd64 9 | - linux_arm64 10 | - linux_arm 11 | - windows_amd64 12 | flags: 13 | - -a 14 | - -trimpath 15 | ldflags: 16 | - -s -w -X main.version={{.Version}} -X main.githead={{.ShortCommit}} -extldflags "-static" 17 | env: 18 | - CGO_ENABLED=0 19 | 20 | archives: 21 | - format: tar.gz 22 | name_template: >- 23 | {{ .ProjectName }}_ 24 | {{- .Os }}_ 25 | {{- if eq .Arch "amd64" }}x86_64 26 | {{- else if eq .Arch "386" }}i386 27 | {{- else }}{{ .Arch }}{{ end }} 28 | {{- if .Arm }}v{{ .Arm }}{{ end }} 29 | format_overrides: 30 | - goos: windows 31 | format: zip 32 | files: 33 | - README.md 34 | - LICENSE 35 | wrap_in_directory: true 36 | 37 | changelog: 38 | sort: asc 39 | filters: 40 | exclude: 41 | - "^docs:" 42 | - "^test:" 43 | 44 | release: 45 | draft: true 46 | github: 47 | owner: rapidloop 48 | name: ellycache 49 | name_template: 'Release {{.Version}}' 50 | checksum: -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ellycache is a simple, performant, query cache for PostgreSQL with a built-in 3 | HTTP server. It runs the queries you specify at cron-like schedules, caches the 4 | results in-memory or on-disk and serves them at specified HTTP endpoints using 5 | the built-in webserver. 6 | 7 | ellycache was built to balance the needs of BI/analytics apps that access 8 | PostgreSQL data, and PostgreSQL DBAs who need to manage the system resources 9 | consumed by PostgreSQL. The cron-like scheduling of SQL queries, together with 10 | a compact single binary deployment that includes an HTTP server, connection 11 | pooler and on-disk encrypted file cache make ellycache a compelling alternative 12 | to traditional query caching software. 13 | 14 | You can start ellycache with a configuration file listing the HTTP URI endpoints 15 | together with the SQL query and run schedule. Here is an example configuration 16 | file: 17 | 18 | ```hcl 19 | listen = ":8080" # host and port for the HTTP server to listen on 20 | 21 | # define the postgres server to connect to. ellycache includes a 22 | # built-in Postgres connection pooler. 23 | connection { 24 | dsn = "host=10.2.2.1 port=5433 user=analyticsro dbname=pagila" 25 | maxconns = 5 26 | idletimeout = "10m" 27 | } 28 | 29 | # define a URI that you can do HTTP GET from, which will run the SQL query 30 | endpoint "/rentals/late" { 31 | sql = < cache.cfg 114 | ``` 115 | 116 | Edit the config file as required, then start ellycache with it: 117 | 118 | ``` 119 | $ ellycache cache.cfg 120 | ``` 121 | 122 | Include the `-d/--debug` flag to see what ellycache is doing: 123 | 124 | ``` 125 | $ ellycache --debug cache.cfg 126 | ``` 127 | 128 | ### Features 129 | 130 | #### Isolation of HTTP requests and PostgreSQL load 131 | 132 | ellycache deliberately does not query on first request and then cache the result 133 | for later requests. By running SQL queries only as per a fixed schedule, the 134 | load on the PostgreSQL server, tracking of long running queries, usage of 135 | resources by expensive analytics queries all become easier to predict and 136 | manage. 137 | 138 | The results of the last expensive analytics query are available to BI apps and 139 | the like until it becomes reasonable to do another query. The freshness of the 140 | data can be seen from the `Last-Modified` header. 141 | 142 | This sets an (adjustable) balance between BI/analytics web apps needs and 143 | database server load management. 144 | 145 | #### Simple configuration 146 | 147 | ellycache is configured with a simple nginx-like configuration file. An example 148 | configuration file can be printed out with `ellycache -e` to serve as both a 149 | starting point and documentation. 150 | 151 | #### Easy to deploy 152 | 153 | ellycache is a single static binary with no dependencies. You can deploy it into 154 | VMs or containers or even bundle it with your apps easily. It is written in 155 | pure Go, and can be built for any platform that [Go supports](https://go.dev/wiki/MinimumRequirements). 156 | 157 | #### PostgreSQL connection pooling 158 | 159 | ellycache maintains an internal PostgreSQL connection pool to limit resource 160 | usage by concurrent queries and reuse connections between quickly repeating 161 | queries. You can configure the *maximum concurrent connections* and 162 | the *maximum duration that a connection is allowed to be idle before closing 163 | it* in the ellycache configuration file. 164 | 165 | #### Encrypted file-backed caches if needed 166 | 167 | If the result of a query is too large and memory usage is a concern, you can ask 168 | ellycache to save the result to a file instead, on a per-endpoint basis. Such 169 | files are AES-256-GCM encrypted with an ephermal key that is valid only for the 170 | current ellycache process lifetime. Only the ellycache process that created the 171 | file can decrypt them, and if that process crashes, then no one can. 172 | 173 | ### Support 174 | 175 | ellycache is an open-source project from [RapidLoop](https://rapidloop.com), 176 | the makers of [pgDash](https://pgdash.io). It is 177 | currently hosted at [GitHub](https://github.com/rapidloop/ellycache). Community 178 | support is available via 179 | [discussions](https://github.com/rapidloop/ellycache/discussions). 180 | Feel free to [raise issues you encounter](https://github.com/rapidloop/ellycache/issues) or 181 | [discuss improvements](https://github.com/rapidloop/ellycache/discussions). For 182 | paid support or custom features for ellycache, do [contact us](mailto:hello@rapidloop.com). 183 | -------------------------------------------------------------------------------- /example.cfg: -------------------------------------------------------------------------------- 1 | # This is an example ellycache configuration file with all possible options. 2 | # Use this as a starting point to create your own configuration file. 3 | 4 | # Syntax of this file: 5 | # * The file itself is in HCL (https://github.com/hashicorp/hcl) format. 6 | # * Strings are enclosed in double quotes, and you can use escape sequences 7 | # like \", \n etc. Heredoc syntax can also be used. 8 | # * Durations are of the format "NNhNNmNNs" where h, m and s stand for hours, 9 | # minutes and seconds respectively, with at least one of them present. 10 | 11 | # The address to listen on. The format is "host:port", "ipv4:port" or 12 | # "[ipv6]:port". Omitting the host binds to all interfaces. The default is 13 | # ":8080". 14 | listen = ":8080" 15 | #listen = "127.0.0.1:8080" 16 | #listen = "[::1]:8080" 17 | #listen = "10.0.23.1:8081" 18 | 19 | # The connection block defines parameters for the PostgreSQL connection pool. 20 | # This block is required. 21 | connection { 22 | # The PostgreSQL DSN to connect to. The format is libpq-style key-value 23 | # pairs separated by spaces. There is no default. ellycache understands 24 | # pgpass files and PG* environment variables. 25 | dsn = "host=myhost port=myport user=myuser password=mypass dbname=mydb" 26 | #dsn = "host=/var/run/postgresql port=5432 user=analytics dbname=posts" 27 | #dsn = "host=pgbouncer.example.com port=6432 dbname=foobar" 28 | 29 | # The maximum number of concurrent active connections. Optional, defaults 30 | # to 5. 31 | maxconns = 10 32 | 33 | # The maximum duration a connection can be idle before it is closed. If 34 | # omitted, idle connections are NOT closed. 35 | idletimeout = "5m" 36 | } 37 | 38 | # The endpoint block defines a single URI served by ellycache. At least one 39 | # endpoint block is required. The URI path should be absolute, and specified 40 | # as a string parameter for the block. 41 | endpoint "/foobar" { 42 | # The SQL command, whose results are cached and served at this URI. This 43 | # is a required parameter. 44 | sql = "SELECT * FROM foobar" 45 | 46 | # The query timeout for the SQL query above. Optional. If not specified, 47 | # no timeout is enforced by ellycache. 48 | sqltimeout = "1m" 49 | 50 | # The schedule to execute this query, in standard cron syntax specified 51 | # here: https://en.wikipedia.org/wiki/Cron. The '@' format ("@every 1h" etc) 52 | # is also supported. This is a required parameter. 53 | schedule = "*/10 * * * *" 54 | 55 | # Each row can be represented as an array or an object in the returned json. 56 | # Use "array" (the default) or "object" as the value for this parameter. 57 | rowformat = "array" 58 | 59 | # If the filebacked parameter is set to true, then the result is not cached 60 | # in memory, but stored in an encrypted temporary file instead. Use this for 61 | # large query results. Optional, defaults to false. 62 | filebacked = true 63 | } 64 | 65 | # Another endpoint block, showing a SQL query in heredoc syntax, and a schedule 66 | # in the '@' format. 67 | endpoint "/pagila/rentals/late" { 68 | sql = < cache.cfg # write an example config to cache.cfg 77 | ellycache cache.cfg # start ellycache with cache.cfg config file 78 | 79 | `) 80 | pflag.PrintDefaults() 81 | fmt.Fprintln(r) 82 | printVersion(r) 83 | } 84 | 85 | func printVersion(r io.Writer) { 86 | fmt.Fprintf(r, "ellycache %s %s\n(c) RapidLoop, Inc. 2024 * https://github.com/rapidloop/ellycache\n", 87 | version, githead) 88 | } 89 | 90 | func main() { 91 | os.Exit(realmain()) 92 | } 93 | 94 | func realmain() int { 95 | // parse command line args 96 | pflag.BoolVarP(&flagVersion, "version", "v", false, "Print version and exit") 97 | pflag.BoolVarP(&flagExample, "example", "e", false, "Print example configuration to stdout and exit") 98 | pflag.BoolVarP(&flagHelp, "help", "h", false, "Print this help message") 99 | pflag.BoolVarP(&flagDebug, "debug", "d", false, "Enable debug logging") 100 | pflag.CommandLine.SortFlags = false 101 | pflag.Usage = func() { printUsage(os.Stderr) } 102 | pflag.Parse() 103 | if flagVersion { 104 | printVersion(os.Stdout) 105 | return 0 106 | } 107 | if flagExample { 108 | io.WriteString(os.Stdout, exampleCfg) 109 | return 0 110 | } 111 | if flagHelp { 112 | printUsage(os.Stdout) 113 | return 0 114 | } 115 | if pflag.NArg() != 1 { 116 | printUsage(os.Stderr) 117 | return 1 118 | } 119 | 120 | // read and validate config file 121 | cfgFile := pflag.Arg(0) 122 | cfg, err := ConfigFromFile(cfgFile) 123 | if err != nil { 124 | fmt.Fprintf(os.Stderr, "ellycache: error reading config file %s: %v\n", cfgFile, err) 125 | return 1 126 | } 127 | if err := cfg.Validate(); err != nil { 128 | fmt.Fprintf(os.Stderr, "ellycache: error in config file %s: %v\n", cfgFile, err) 129 | return 1 130 | } 131 | 132 | // create encryption key 133 | if cryptKey, err = symmecrypt.NewRandomKey("aes-gcm"); err != nil { 134 | fmt.Fprintf(os.Stderr, "ellycache: failed to create encryption key: %v\n", err) 135 | return 1 136 | } 137 | 138 | // connect to postgres 139 | if pgcfg, err := makePgxCfg(&cfg.Connection); err != nil { 140 | fmt.Fprintf(os.Stderr, "ellycache: error in connection config: %v\n", err) 141 | return 1 142 | } else if pool, err = pgxpool.NewWithConfig(context.Background(), pgcfg); err != nil { 143 | fmt.Fprintf(os.Stderr, "ellycache: failed to connect: %v\n", err) 144 | return 1 145 | } 146 | defer pool.Close() 147 | 148 | // start cron 149 | crond = cron.New() 150 | for _, e := range cfg.Endpoints { 151 | job := &Job{endpoint: &e} 152 | if entryID, err := crond.AddJob(e.Schedule, job); err != nil { 153 | fmt.Fprintf(os.Stderr, "ellycache: bad schedule \"%s\" in endpoint \"%s\"\n", e.Schedule, e.Path) 154 | return 1 155 | } else { 156 | job.entryID = entryID 157 | if flagDebug { 158 | if s, err := cron.ParseStandard(e.Schedule); err == nil { 159 | log.Printf("debug: scheduled %s, next at %s", e.Path, 160 | s.Next(time.Now()).Format("2006-01-02 15:04:05")) 161 | } 162 | } 163 | } 164 | } 165 | crond.Start() 166 | defer func() { 167 | ctx := crond.Stop() 168 | <-ctx.Done() 169 | }() 170 | 171 | // configure an http mux with the endpoints, wrap in compress handler 172 | mux := http.NewServeMux() 173 | for _, e := range cfg.Endpoints { 174 | mux.HandleFunc("GET "+e.Path, httpHandler) 175 | } 176 | handler := handlers.CompressHandler(mux) 177 | 178 | // start an http server 179 | listen := strings.TrimSpace(cfg.Listen) 180 | if listen == "" { 181 | listen = defaultListen 182 | } 183 | srv := &http.Server{ 184 | Addr: listen, 185 | Handler: handler, 186 | } 187 | if err := srv.ListenAndServe(); err != nil { 188 | if !errors.Is(err, http.ErrServerClosed) { 189 | fmt.Fprintf(os.Stderr, "ellycache: http server error: %v\n", err) 190 | return 1 191 | } 192 | } 193 | 194 | return 0 195 | } 196 | 197 | // makePgxCfg creates a pgxpool.Config from a ConnectionConfig. 198 | func makePgxCfg(c *ConnectionConfig) (*pgxpool.Config, error) { 199 | cfg, err := pgxpool.ParseConfig(c.DSN) 200 | if err != nil { 201 | return nil, err 202 | } 203 | if c.MaxConns > 0 { 204 | cfg.MaxConns = int32(c.MaxConns) 205 | } else { 206 | cfg.MaxConns = defaultMaxConns 207 | } 208 | if d, err := time.ParseDuration(c.IdleTimeout); err == nil && d > 0 { 209 | cfg.MaxConnIdleTime = d 210 | } 211 | cfg.ConnConfig.DefaultQueryExecMode = pgx.QueryExecModeSimpleProtocol 212 | return cfg, nil 213 | } 214 | 215 | // Job is the cron job handler for an endpoint. 216 | type Job struct { 217 | entryID cron.EntryID 218 | endpoint *EndpointConfig 219 | } 220 | 221 | // Run is the cron job handler for an endpoint. If the query is successful, it 222 | // updates the cache. 223 | func (j *Job) Run() { 224 | // helper to delete old result's file 225 | cleanOld := func(oldResult *EndpointResult) { 226 | if oldResult.File != "" { 227 | if err := os.Remove(oldResult.File); err != nil { 228 | log.Printf("warning: failed to remove file %s: %v", oldResult.File, err) 229 | } 230 | } 231 | } 232 | 233 | // make new result 234 | newResult, err := makeResult(j.endpoint) 235 | if err != nil { 236 | log.Printf("warning: failed to query for %s: %v", j.endpoint.Path, err) 237 | if oldResult, loaded := cache.LoadAndDelete(j.endpoint.Path); loaded { 238 | or, _ := oldResult.(*EndpointResult) 239 | cleanOld(or) 240 | log.Printf("warning: removed old result for %s", j.endpoint.Path) 241 | } 242 | return 243 | } 244 | newResult.QueriedAt = time.Now().Truncate(time.Second).In(time.UTC).Format(http.TimeFormat) 245 | newResult.CacheControl = fmt.Sprintf("max-age=%d, immutable", 246 | int(crond.Entry(j.entryID).Next.Sub(time.Now()).Seconds())) 247 | 248 | // update cache, cleanup old entry if present 249 | oldResult, loaded := cache.Swap(j.endpoint.Path, newResult) 250 | if loaded { 251 | or, _ := oldResult.(*EndpointResult) 252 | cleanOld(or) 253 | if flagDebug { 254 | if or.ETag == newResult.ETag { 255 | log.Printf("debug: replaced result for %s (no change in content)", j.endpoint.Path) 256 | } else { 257 | log.Printf("debug: replaced result for %s", j.endpoint.Path) 258 | } 259 | } 260 | } else if flagDebug { 261 | log.Printf("debug: populated result for %s", j.endpoint.Path) 262 | } 263 | } 264 | 265 | // makeResult performs the query for the endpoint and returns the result. 266 | func makeResult(e *EndpointConfig) (*EndpointResult, error) { 267 | if e.FileBacked { 268 | if f, err := os.CreateTemp("", "ellycache"); err != nil { 269 | return nil, fmt.Errorf("failed to create temp file: %v", err) 270 | } else { 271 | defer f.Close() 272 | w := symmecrypt.NewWriter(f, cryptKey) 273 | defer w.Close() 274 | if h, err := query(e, w); err != nil { 275 | return nil, err 276 | } else { 277 | return &EndpointResult{ETag: fmt.Sprintf(`W/"%x"`, h), File: f.Name()}, nil 278 | } 279 | } 280 | } 281 | 282 | b := &bytes.Buffer{} 283 | if h, err := query(e, b); err != nil { 284 | return nil, err 285 | } else { 286 | return &EndpointResult{ETag: fmt.Sprintf(`W/"%x"`, h), Result: b.Bytes()}, nil 287 | } 288 | } 289 | 290 | // query runs the SQL query for and endpoint and writes the result to the given 291 | // io.Writer. 292 | func query(e *EndpointConfig, w io.Writer) (uint64, error) { 293 | t1 := time.Now() 294 | defer func() { 295 | if flagDebug { 296 | log.Printf("debug: database query for %s took %v", e.Path, time.Since(t1)) 297 | } 298 | }() 299 | 300 | digest := xxhash.New() 301 | 302 | ctx := context.Background() 303 | if d, err := time.ParseDuration(e.SQLTimeout); err == nil && d > 0 { 304 | var cancel context.CancelFunc 305 | ctx, cancel = context.WithTimeout(ctx, d) 306 | defer cancel() 307 | } 308 | 309 | rows, err := pool.Query(ctx, e.SQL) 310 | if err != nil { 311 | return 0, fmt.Errorf("query failed: %v", err) 312 | } 313 | defer rows.Close() 314 | 315 | var columns []string 316 | if e.RowFormat != "array" { 317 | for _, fd := range rows.FieldDescriptions() { 318 | columns = append(columns, fd.Name) 319 | } 320 | } 321 | 322 | first := true 323 | fmt.Fprintln(w, "[") 324 | for rows.Next() { 325 | values, err := rows.Values() 326 | if err != nil { 327 | return 0, fmt.Errorf("query read failed: %v", err) 328 | } 329 | var row any 330 | if e.RowFormat == "array" { 331 | row = values 332 | } else { 333 | m := make(map[string]any) 334 | for i, v := range values { 335 | m[columns[i]] = v 336 | } 337 | row = m 338 | } 339 | j, err := json.Marshal(row) 340 | if err != nil { 341 | return 0, fmt.Errorf("failed to marshal to json: %v", err) 342 | } 343 | digest.Write(j) 344 | if first { 345 | first = false 346 | } else { 347 | fmt.Fprintln(w, ",") 348 | } 349 | fmt.Fprint(w, " ") 350 | fmt.Fprint(w, string(j)) 351 | } 352 | if err := rows.Err(); err != nil { 353 | return 0, fmt.Errorf("query error: %v", err) 354 | } 355 | fmt.Fprintln(w) 356 | fmt.Fprintln(w, "]") 357 | return digest.Sum64(), nil 358 | } 359 | 360 | // httpHandler serves up content for an endpoint. 361 | func httpHandler(w http.ResponseWriter, r *http.Request) { 362 | hdr := w.Header() 363 | code := 200 364 | t1 := time.Now() 365 | defer func() { 366 | if flagDebug { 367 | log.Printf("debug: %q %d %v", r.URL.Path, code, time.Since(t1)) 368 | } 369 | }() 370 | 371 | // load result 372 | v, ok := cache.Load(r.URL.Path) 373 | if !ok || v == nil { 374 | w.Header().Set("Cache-Control", "no-cache, no-store") 375 | code = http.StatusNotFound 376 | http.NotFound(w, r) 377 | return 378 | } 379 | result, _ := v.(*EndpointResult) 380 | sethdr := func() { 381 | hdr.Set("ETag", result.ETag) 382 | hdr.Set("Last-Modified", result.QueriedAt) 383 | hdr.Set("Cache-Control", result.CacheControl) 384 | } 385 | 386 | // check etag 387 | if r.Header.Get("If-None-Match") == result.ETag { 388 | code = http.StatusNotModified 389 | sethdr() 390 | w.WriteHeader(http.StatusNotModified) 391 | return 392 | } 393 | 394 | // serve from memory 395 | if result.File == "" { 396 | hdr.Set("Content-Type", "application/json") 397 | sethdr() 398 | w.Write(result.Result) 399 | return 400 | } 401 | 402 | // serve from file 403 | f, err := os.Open(result.File) 404 | if err != nil { 405 | hdr.Set("Cache-Control", "no-cache, no-store") 406 | code = http.StatusInternalServerError 407 | log.Printf("http: %s: failed to open file %s: %v", r.URL.Path, result.File, err) 408 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 409 | return 410 | } 411 | defer f.Close() 412 | fr, err := symmecrypt.NewReader(f, cryptKey) 413 | if err != nil { 414 | hdr.Set("Cache-Control", "no-cache, no-store") 415 | code = http.StatusInternalServerError 416 | log.Printf("http: %s: failed to decrypt file %s: %v", r.URL.Path, result.File, err) 417 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 418 | return 419 | } 420 | hdr.Set("Content-Type", "application/json") 421 | sethdr() 422 | if _, err := io.Copy(w, fr); err != nil { 423 | code = http.StatusInternalServerError 424 | log.Printf("http: %s: failed to write response: %v", r.URL.Path, err) 425 | http.Error(w, "Internal Server Error", http.StatusInternalServerError) 426 | return 427 | } 428 | } 429 | -------------------------------------------------------------------------------- /model.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "net" 23 | "os" 24 | "regexp" 25 | "strings" 26 | "time" 27 | 28 | "github.com/hashicorp/hcl/v2" 29 | "github.com/hashicorp/hcl/v2/gohcl" 30 | "github.com/hashicorp/hcl/v2/hclsyntax" 31 | "github.com/jackc/pgx/v5" 32 | "github.com/robfig/cron/v3" 33 | ) 34 | 35 | type Config struct { 36 | Listen string `hcl:"listen,optional"` 37 | Connection ConnectionConfig `hcl:"connection,block"` 38 | Endpoints []EndpointConfig `hcl:"endpoint,block"` 39 | } 40 | 41 | func (c *Config) Validate() error { 42 | if _, _, err := net.SplitHostPort(c.Listen); err != nil { 43 | return fmt.Errorf("listen: %v", err) 44 | } 45 | if err := c.Connection.Validate(); err != nil { 46 | return err 47 | } 48 | if len(c.Endpoints) == 0 { 49 | return errors.New("no endpoints defined") 50 | } 51 | for _, e := range c.Endpoints { 52 | if err := e.Validate(); err != nil { 53 | return err 54 | } 55 | } 56 | return nil 57 | } 58 | 59 | type ConnectionConfig struct { 60 | DSN string `hcl:"dsn"` 61 | MaxConns int `hcl:"maxconns,optional"` 62 | IdleTimeout string `hcl:"idletimeout,optional"` 63 | } 64 | 65 | func (c *ConnectionConfig) Validate() error { 66 | if _, err := pgx.ParseConfig(c.DSN); err != nil { 67 | return fmt.Errorf("connection.dsn: %v", err) 68 | } 69 | if c.MaxConns < 0 { 70 | return errors.New("connection.maxconns: cannot be negative") 71 | } 72 | if c.IdleTimeout != "" { 73 | if _, err := time.ParseDuration(c.IdleTimeout); err != nil { 74 | return fmt.Errorf("connection.idletimeout: %v", err) 75 | } 76 | } 77 | return nil 78 | } 79 | 80 | type EndpointConfig struct { 81 | Path string `hcl:"path,label"` 82 | SQL string `hcl:"sql"` 83 | SQLTimeout string `hcl:"sqltimeout,optional"` 84 | Schedule string `hcl:"schedule"` 85 | RowFormat string `hcl:"rowformat,optional"` 86 | FileBacked bool `hcl:"filebacked,optional"` 87 | } 88 | 89 | var rxURI = regexp.MustCompile(`^(/(({[A-Za-z0-9_.-]+})|([A-Za-z0-9_.-]+)))+$`) 90 | 91 | func (e *EndpointConfig) Validate() error { 92 | if !rxURI.MatchString(e.Path) && e.Path != "/" { 93 | return fmt.Errorf("endpoint \"%s\": invalid path: must be set to a valid URI", e.Path) 94 | } 95 | if strings.TrimSpace(e.SQL) == "" { 96 | return fmt.Errorf("endpoint \"%s\": invalid sql: must be set", e.Path) 97 | } 98 | if e.SQLTimeout != "" { 99 | if _, err := time.ParseDuration(e.SQLTimeout); err != nil { 100 | return fmt.Errorf("endpoint \"%s\": invalid sqltimeout: %v", e.Path, err) 101 | } 102 | } 103 | if _, err := cron.ParseStandard(e.Schedule); err != nil { 104 | return fmt.Errorf("endpoint \"%s\": invalid schedule: %v", e.Path, err) 105 | } 106 | if e.RowFormat != "" && e.RowFormat != "array" && e.RowFormat != "object" { 107 | return fmt.Errorf("endpoint \"%s\": invalid rowformat: must be 'array' or 'object'", e.Path) 108 | } 109 | return nil 110 | } 111 | 112 | func ConfigFromFile(filename string) (*Config, error) { 113 | var cfg Config 114 | src, err := os.ReadFile(filename) 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | file, diags := hclsyntax.ParseConfig(src, filename, hcl.Pos{Line: 1, Column: 1}) 120 | if diags.HasErrors() { 121 | return nil, diags 122 | } 123 | 124 | diags = gohcl.DecodeBody(file.Body, nil, &cfg) 125 | if diags.HasErrors() { 126 | return nil, diags 127 | } 128 | return &cfg, nil 129 | } 130 | 131 | //------------------------------------------------------------------------------ 132 | 133 | type EndpointResult struct { 134 | QueriedAt string // as a string to avoid formatting each time 135 | CacheControl string 136 | ValidUntil time.Time 137 | ETag string 138 | Result []byte 139 | File string // if not empty, result is in this file 140 | } 141 | --------------------------------------------------------------------------------