├── .gitignore ├── .goreleaser.yml ├── LICENSE ├── README.md ├── _examples ├── cache.yaml ├── commonprefix.json ├── compression.yaml ├── cors.yaml ├── datasources-lazy.json ├── datasources-pool.json ├── datasources-role.json ├── datasources-ssl.json ├── exec-debug.yaml ├── exec-in-tx.yaml ├── exec.yaml ├── javascript-result.yaml ├── javascript.yaml ├── job-partition.yaml ├── job-refresh-sales-every-format.json ├── job-refresh-sales.json ├── listen.yaml ├── methods.json ├── param-in-body.yaml ├── params-arrays.yaml ├── params-enum.json ├── params-enum.yaml ├── params-json.yaml ├── params-minmax.yaml ├── params-required.yaml ├── params-strings.yaml ├── query-simple-select-csv.yaml ├── query-simple-select.yaml ├── query-timeout.yaml ├── query-with-param.yaml ├── static-json.yaml ├── static-text.json ├── static-text.yaml ├── streams-sse.yaml ├── streams-websockets.yaml └── ws-client.html ├── _test ├── invalid_cfgs.jsons └── warn_cfgs.jsons ├── cmd └── rapidrows │ └── main.go ├── datasources.go ├── datasources_test.go ├── doc.go ├── go.mod ├── go.sum ├── jobs.go ├── jobs_test.go ├── model.go ├── params.go ├── params_test.go ├── qjs ├── cutils.c ├── cutils.h ├── libbf.c ├── libbf.h ├── libregexp-opcode.h ├── libregexp.c ├── libregexp.h ├── libunicode-table.h ├── libunicode.c ├── libunicode.h ├── list.h ├── quickjs-atom.h ├── quickjs-opcode.h ├── quickjs.c ├── quickjs.h ├── wrap.go └── wrap.h ├── script.go ├── script_test.go ├── server.go ├── server_test.go ├── streams.go ├── streams_test.go ├── validate.go └── validate_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | /rapidrows 2 | /dist/ 3 | /cover.out 4 | /cover.html -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - binary: rapidrows 3 | main: ./cmd/rapidrows 4 | targets: 5 | - linux_amd64 6 | flags: 7 | - -a 8 | - -trimpath 9 | ldflags: 10 | - -s -w -X main.version={{.Version}} -extldflags "-static" 11 | archives: 12 | - format: tar.gz 13 | format_overrides: 14 | - goos: windows 15 | format: zip 16 | files: 17 | - README.md 18 | - LICENSE 19 | wrap_in_directory: true 20 | name_template: '{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}' 21 | release: 22 | draft: true 23 | github: 24 | owner: rapidloop 25 | name: rapidrows 26 | name_template: 'Release {{.Version}}' 27 | checksum: 28 | 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | RapidRows is an open-source, zero-dependency, single-binary API server that can 2 | be configured to run SQL queries, perform scheduled jobs and forward PostgreSQL 3 | notifications to websockets. 4 | 5 | For more information, see [rapidrows.io](https://rapidrows.io). 6 | 7 | RapidRows is developed and maintained by [RapidLoop](https://rapidloop.com). 8 | Follow us on Twitter at [@therapidloop](https://twitter.com/therapidloop/). 9 | 10 | RapidRows is licensed under the Apache 2.0 open source license, copyright 11 | (c) RapidLoop, Inc. 2022. It includes source code from QuickJS Javascript 12 | Engine, licensed under the MIT license and is copyrighted (c) Fabrice Bellard 13 | and Charlie Gordon. 14 | -------------------------------------------------------------------------------- /_examples/cache.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /param-in-body 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT title, description FROM film WHERE fulltext @@ to_tsquery($1) ORDER BY title ASC 8 | params: 9 | - name: descfts 10 | in: body 11 | type: string 12 | required: true 13 | cache: 3600 14 | datasources: 15 | - name: pagila 16 | dbname: pagila 17 | -------------------------------------------------------------------------------- /_examples/commonprefix.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "commonPrefix": "/api/v1", 4 | "endpoints": [ 5 | { 6 | "uri": "/hello", 7 | "implType": "static-text", 8 | "script": "world!" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /_examples/compression.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | compression: true 3 | endpoints: 4 | - uri: '/hello-compressed' 5 | implType: 'static-text' 6 | script: | 7 | Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod 8 | tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, 9 | quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo 10 | consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse 11 | cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non 12 | proident, sunt in culpa qui officia deserunt mollit anim id est laborum. 13 | -------------------------------------------------------------------------------- /_examples/cors.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | cors: 3 | allowedOrigins: [ 'http://foo.com' ] 4 | allowedMethods: [ 'GET' ] 5 | allowedHeaders: [ 'X-Pingother', 'Content-Type' ] 6 | debug: true 7 | endpoints: 8 | - uri: /hello 9 | implType: static-text 10 | script: world! 11 | -------------------------------------------------------------------------------- /_examples/datasources-lazy.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "datasources": [ 4 | { 5 | "name": "pagila", 6 | "pool": { 7 | "minConns": 5, 8 | "maxConns": 10, 9 | "maxIdleTime": 600, 10 | "lazy": true 11 | } 12 | } 13 | ] 14 | } 15 | 16 | -------------------------------------------------------------------------------- /_examples/datasources-pool.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "datasources": [ 4 | { 5 | "name": "pagila", 6 | "pool": { 7 | "minConns": 5, 8 | "maxConns": 10, 9 | "maxIdleTime": 600 10 | } 11 | } 12 | ] 13 | } 14 | -------------------------------------------------------------------------------- /_examples/datasources-role.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "datasources": [ 4 | { 5 | "name": "pagila", 6 | "user": "postgres", 7 | "role": "readonlyuser" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /_examples/datasources-ssl.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "datasources": [ 4 | { 5 | "name": "pagila", 6 | "host": "pg-staging-3.corp.example.com", 7 | "dbname": "pagila", 8 | "user": "webappuser", 9 | "passfile": "/home/deploy/.pgpass", 10 | "sslmode": "require", 11 | "params": { 12 | "application_name": "rapidrows", 13 | "search_path": "pagila,public" 14 | }, 15 | "timeout": 10 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /_examples/exec-debug.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /exec-debug 4 | implType: exec 5 | methods: 6 | - POST 7 | datasource: pagila 8 | script: UPDATE rental SET return_date = now() WHERE rental_id = $1 9 | params: 10 | - name: rental_id 11 | in: body 12 | type: integer 13 | minimum: 1 14 | required: true 15 | debug: true 16 | datasources: 17 | - name: pagila 18 | dbname: pagila 19 | -------------------------------------------------------------------------------- /_examples/exec-in-tx.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /exec 4 | implType: exec 5 | methods: 6 | - POST 7 | datasource: pagila 8 | script: UPDATE rental SET return_date = now() WHERE rental_id = $1 9 | params: 10 | - name: rental_id 11 | in: body 12 | type: integer 13 | minimum: 1 14 | required: true 15 | tx: 16 | access: 'read write' 17 | level: 'serializable' 18 | deferrable: false 19 | datasources: 20 | - name: pagila 21 | dbname: pagila 22 | -------------------------------------------------------------------------------- /_examples/exec.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /exec 4 | implType: exec 5 | methods: 6 | - POST 7 | datasource: pagila 8 | script: UPDATE rental SET return_date = now() WHERE rental_id = $1 9 | params: 10 | - name: rental_id 11 | in: body 12 | type: integer 13 | minimum: 1 14 | required: true 15 | datasources: 16 | - name: pagila 17 | dbname: pagila 18 | -------------------------------------------------------------------------------- /_examples/javascript-result.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /rental/{rental_id} 4 | implType: javascript 5 | datasource: pagila 6 | script: | 7 | // get a connection to a datasource 8 | let conn = $sys.acquire("pagila"); 9 | 10 | // perform a query 11 | let queryResult = conn.query(` 12 | select C.first_name || ' ' || C.last_name 13 | from rental R 14 | join customer C on R.customer_id = C.customer_id 15 | where R.rental_id = $1 16 | `, $sys.params.rental_id); 17 | 18 | // check the result 19 | if (queryResult.rows.length != 1) 20 | throw "Rental not found"; 21 | const custname = queryResult.rows[0][0]; 22 | 23 | // return a result 24 | $sys.result = { 'custname': custname }; 25 | params: 26 | - name: rental_id 27 | in: path 28 | type: integer 29 | minimum: 1 30 | required: true 31 | datasources: 32 | - name: pagila 33 | dbname: pagila 34 | -------------------------------------------------------------------------------- /_examples/javascript.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /exec-js 4 | implType: javascript 5 | methods: 6 | - POST 7 | datasource: pagila 8 | script: | 9 | // get a connection to a datasource 10 | let conn = $sys.acquire("pagila"); 11 | 12 | // perform a query 13 | let genreResult = conn.query(` 14 | select C.name 15 | from rental R 16 | join inventory I on R.inventory_id = I.inventory_id 17 | join film_category FC on I.film_id = FC.film_id 18 | join category C on C.category_id = FC.category_id 19 | where R.rental_id = $1 20 | `, $sys.params.rental_id); 21 | 22 | // check the result 23 | if (genreResult.rows.length != 1) 24 | throw "Rental not found"; 25 | const genre = genreResult.rows[0][0]; 26 | 27 | // further checks 28 | let today = (new Date()).getDay(); 29 | if (genre == "Horror" && today == 3) 30 | throw "Cannot return Horror DVDs on Wednesdays!" 31 | 32 | // exec a SQL without a resultset 33 | conn.exec("UPDATE rental SET return_date = now() WHERE rental_id = $1", 34 | $sys.params.rental_id) 35 | params: 36 | - name: rental_id 37 | in: body 38 | type: integer 39 | minimum: 1 40 | required: true 41 | datasources: 42 | - name: pagila 43 | dbname: pagila 44 | -------------------------------------------------------------------------------- /_examples/job-partition.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | jobs: 3 | - name: create-monthly-partition 4 | type: javascript 5 | schedule: '0 10 28 * *' 6 | datasource: pagila 7 | script: | 8 | const now = new Date(); 9 | const nextMonth = new Date(now.getFullYear(), now.getMonth()+1, 1); 10 | const next2Month = new Date(nextMonth.getFullYear(), nextMonth.getMonth()+1, 1); 11 | const y1 = nextMonth.getFullYear(), m1 = nextMonth.getMonth() + 1; 12 | const y2 = next2Month.getFullYear(), m2 = next2Month.getMonth() + 1; 13 | const m1s = (m1 < 10 ? '0' : '') + m1, m2s = (m2 < 10 ? '0' : '') + m2; 14 | 15 | // make the sql to create a partition for next month 16 | const sql = ` 17 | CREATE TABLE public.payment_p${y1}_${m1s} ( 18 | payment_id integer DEFAULT nextval('public.payment_payment_id_seq'::regclass) NOT NULL, 19 | customer_id integer NOT NULL, 20 | staff_id integer NOT NULL, 21 | rental_id integer NOT NULL, 22 | amount numeric(5,2) NOT NULL, 23 | payment_date timestamp with time zone NOT NULL 24 | ); 25 | 26 | ALTER TABLE ONLY public.payment 27 | ATTACH PARTITION public.payment_p${y1}_${m1s} 28 | FOR VALUES FROM ('${y1}-${m1s}-01 00:00:00+00') TO ('${y2}-${m2s}-01 00:00:00+00');` 29 | 30 | // run the sql 31 | $sys.acquire('pagila').exec(sql) 32 | datasources: 33 | - dbname: pagila 34 | name: pagila 35 | -------------------------------------------------------------------------------- /_examples/job-refresh-sales-every-format.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "jobs": [ 4 | { 5 | "name": "refresh-sales-every-format", 6 | "schedule": "@every 1h", 7 | "type": "exec", 8 | "script": "REFRESH MATERIALIZED VIEW rental_by_category", 9 | "datasource": "pagila", 10 | "debug": true 11 | } 12 | ], 13 | "datasources": [ 14 | { 15 | "name": "pagila", 16 | "dbname": "pagila" 17 | } 18 | ] 19 | } 20 | -------------------------------------------------------------------------------- /_examples/job-refresh-sales.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "jobs": [ 4 | { 5 | "name": "refresh-sales", 6 | "schedule": "0 * * * *", 7 | "type": "exec", 8 | "script": "REFRESH MATERIALIZED VIEW rental_by_category", 9 | "datasource": "pagila" 10 | } 11 | ], 12 | "datasources": [ 13 | { 14 | "name": "pagila", 15 | "dbname": "pagila" 16 | } 17 | ] 18 | } 19 | -------------------------------------------------------------------------------- /_examples/listen.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | listen: '127.0.0.1:8081' 3 | endpoints: 4 | - uri: /hello 5 | implType: static-text 6 | script: world! 7 | -------------------------------------------------------------------------------- /_examples/methods.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "endpoints": [ 4 | { 5 | "uri": "/hello", 6 | "methods": [ "POST", "PUT" ], 7 | "implType": "static-text", 8 | "script": "world!" 9 | } 10 | ] 11 | } 12 | -------------------------------------------------------------------------------- /_examples/param-in-body.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /param-in-body 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT title, description FROM film WHERE fulltext @@ to_tsquery($1) ORDER BY title ASC 8 | params: 9 | - name: descfts 10 | in: body 11 | type: string 12 | required: true 13 | datasources: 14 | - name: pagila 15 | dbname: pagila 16 | -------------------------------------------------------------------------------- /_examples/params-arrays.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /params-arrays 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT $1 8 | params: 9 | - name: genres 10 | in: body 11 | type: array 12 | elemType: string 13 | minItems: 1 14 | maxItems: 5 15 | datasources: 16 | - name: pagila 17 | dbname: pagila 18 | -------------------------------------------------------------------------------- /_examples/params-enum.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "endpoints": [ 4 | { 5 | "uri": "/params-enum", 6 | "implType": "query-json", 7 | "datasource": "pagila", 8 | "script": "SELECT $1, $2, $3\n", 9 | "params": [ 10 | { 11 | "name": "lang", 12 | "in": "body", 13 | "type": "string", 14 | "required": true, 15 | "enum": [ 16 | "english", 17 | "german", 18 | "japanese" 19 | ] 20 | }, 21 | { 22 | "name": "year", 23 | "in": "body", 24 | "type": "integer", 25 | "required": true, 26 | "enum": [ 27 | 2022, 28 | 1950 29 | ] 30 | } 31 | ] 32 | } 33 | ], 34 | "datasources": [ 35 | { 36 | "name": "pagila", 37 | "dbname": "pagila" 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /_examples/params-enum.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /params-enum 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT $1, $2 8 | params: 9 | - name: lang 10 | in: body 11 | type: string 12 | required: true 13 | enum: [ english, german, japanese ] 14 | - name: year 15 | in: body 16 | type: integer 17 | required: true 18 | enum: [ 2022, 1950 ] 19 | datasources: 20 | - name: pagila 21 | dbname: pagila 22 | -------------------------------------------------------------------------------- /_examples/params-json.yaml: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "endpoints": [ 4 | { 5 | "uri": "/params-enum", 6 | "implType": "query-json", 7 | "datasource": "pagila", 8 | "script": "SELECT $1, $2\n", 9 | "params": [ 10 | { 11 | "name": "lang", 12 | "in": "body", 13 | "type": "string", 14 | "required": true, 15 | "enum": [ 16 | "english", 17 | "german", 18 | "japanese" 19 | ] 20 | }, 21 | { 22 | "name": "year", 23 | "in": "body", 24 | "type": "integer", 25 | "required": true, 26 | "enum": [ 27 | 2022, 28 | 1950 29 | ] 30 | } 31 | ] 32 | } 33 | ], 34 | "datasources": [ 35 | { 36 | "name": "pagila", 37 | "dbname": "pagila" 38 | } 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /_examples/params-minmax.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /params-minmax 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT $1 8 | params: 9 | - name: year 10 | in: body 11 | type: integer 12 | required: true 13 | minimum: 1972 14 | maximum: 2022 15 | datasources: 16 | - name: pagila 17 | dbname: pagila 18 | -------------------------------------------------------------------------------- /_examples/params-required.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /params-required 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT $1, $2 8 | params: 9 | - name: required_param 10 | in: body 11 | type: string 12 | required: true 13 | - name: optional_param 14 | in: body 15 | type: string 16 | datasources: 17 | - name: pagila 18 | dbname: pagila 19 | -------------------------------------------------------------------------------- /_examples/params-strings.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /params-strings 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT $1 8 | params: 9 | - name: lang 10 | in: body 11 | type: string 12 | maxLength: 10 13 | pattern: '.*(ish|an|ese)' 14 | datasources: 15 | - name: pagila 16 | dbname: pagila 17 | -------------------------------------------------------------------------------- /_examples/query-simple-select-csv.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /query-simple-select-csv 4 | implType: query-csv 5 | datasource: pagila 6 | script: | 7 | SELECT 8 | CONCAT(customer.last_name, ', ', customer.first_name) AS customer, 9 | address.phone, 10 | film.title 11 | FROM 12 | rental 13 | INNER JOIN customer ON rental.customer_id = customer.customer_id 14 | INNER JOIN address ON customer.address_id = address.address_id 15 | INNER JOIN inventory ON rental.inventory_id = inventory.inventory_id 16 | INNER JOIN film ON inventory.film_id = film.film_id 17 | WHERE 18 | rental.return_date IS NULL 19 | AND rental_date < CURRENT_DATE 20 | ORDER BY 21 | title 22 | LIMIT 5 23 | datasources: 24 | - name: pagila 25 | dbname: pagila 26 | 27 | -------------------------------------------------------------------------------- /_examples/query-simple-select.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /query-simple-select 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT 8 | CONCAT(customer.last_name, ', ', customer.first_name) AS customer, 9 | address.phone, 10 | film.title 11 | FROM 12 | rental 13 | INNER JOIN customer ON rental.customer_id = customer.customer_id 14 | INNER JOIN address ON customer.address_id = address.address_id 15 | INNER JOIN inventory ON rental.inventory_id = inventory.inventory_id 16 | INNER JOIN film ON inventory.film_id = film.film_id 17 | WHERE 18 | rental.return_date IS NULL 19 | AND rental_date < CURRENT_DATE 20 | ORDER BY 21 | title 22 | LIMIT 5 23 | datasources: 24 | - name: pagila 25 | dbname: pagila 26 | 27 | -------------------------------------------------------------------------------- /_examples/query-timeout.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /query-timeout 4 | implType: query-json 5 | datasource: pagila 6 | script: SELECT pg_sleep(60) 7 | timeout: 5 8 | datasources: 9 | - name: pagila 10 | dbname: pagila 11 | 12 | -------------------------------------------------------------------------------- /_examples/query-with-param.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /query-with-param 4 | implType: query-json 5 | datasource: pagila 6 | script: | 7 | SELECT title, description FROM film WHERE fulltext @@ to_tsquery($1) ORDER BY title ASC 8 | params: 9 | - name: descfts 10 | in: query 11 | type: string 12 | required: true 13 | datasources: 14 | - name: pagila 15 | dbname: pagila 16 | -------------------------------------------------------------------------------- /_examples/static-json.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /hello-json 4 | implType: static-json 5 | script: > 6 | { "mykey1": 100, "mykey2": [1.2, 1.3], 7 | "mykey3": "hello" } 8 | -------------------------------------------------------------------------------- /_examples/static-text.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1", 3 | "endpoints": [ 4 | { 5 | "uri": "/hello", 6 | "implType": "static-text", 7 | "script": "world!" 8 | } 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /_examples/static-text.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | endpoints: 3 | - uri: /hello 4 | implType: static-text 5 | script: world! 6 | 7 | -------------------------------------------------------------------------------- /_examples/streams-sse.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | streams: 3 | - uri: '/new_payments_sse' 4 | type: 'sse' 5 | datasource: 'pagila' 6 | channel: 'payment_received' 7 | datasources: 8 | - name: 'pagila' 9 | dbname: 'pagila' 10 | -------------------------------------------------------------------------------- /_examples/streams-websockets.yaml: -------------------------------------------------------------------------------- 1 | version: '1' 2 | streams: 3 | - uri: '/new_payments_ws' 4 | type: 'websocket' 5 | datasource: 'pagila' 6 | channel: 'payment_received' 7 | datasources: 8 | - name: 'pagila' 9 | dbname: 'pagila' 10 | -------------------------------------------------------------------------------- /_examples/ws-client.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /_test/warn_cfgs.jsons: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.0.0", 3 | "jobs": [ 4 | { 5 | "name": "a", 6 | "type": "exec", 7 | "script": "asdf", 8 | "schedule": "* */10 * * *", 9 | "datasource": "ds1", 10 | "timeout": 0 11 | } 12 | ], 13 | "datasources": [{"name": "ds1"}] 14 | } 15 | 16 | { 17 | "version": "1.0.0", 18 | "cors": { "maxAge": -1 } 19 | } 20 | 21 | { 22 | "version": "1.0.0", 23 | "endpoints": [ 24 | { 25 | "uri": "/", 26 | "implType": "static-text", 27 | "timeout": -1.5, 28 | "cache": -2.5 29 | } 30 | ] 31 | } 32 | 33 | { 34 | "version": "1.0.0", 35 | "datasources": [{"name": "ds1", "timeout": 0}] 36 | } 37 | -------------------------------------------------------------------------------- /cmd/rapidrows/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "encoding/json" 21 | "fmt" 22 | "log" 23 | "os" 24 | "os/signal" 25 | "sync" 26 | "time" 27 | 28 | "github.com/goccy/go-yaml" 29 | "github.com/mattn/go-isatty" 30 | "github.com/rapidloop/rapidrows" 31 | "github.com/rs/zerolog" 32 | "github.com/spf13/pflag" 33 | ) 34 | 35 | var ( 36 | flagset = pflag.NewFlagSet("", pflag.ContinueOnError) 37 | fversion = flagset.BoolP("version", "v", false, "show version and exit") 38 | fcheck = flagset.BoolP("check", "c", false, "only check if the config file is valid") 39 | flog = flagset.StringP("logtype", "l", "text", "print logs in 'text' (default) or 'json' format") 40 | fnocolor = flagset.Bool("no-color", false, "do not colorize log output") 41 | fyaml = flagset.BoolP("yaml", "y", false, "config-file is in YAML format") 42 | ) 43 | 44 | var version string // set during build 45 | 46 | func usage() { 47 | fmt.Fprintf(os.Stderr, `Usage: rapidrows [options] config-file 48 | RapidRows is a single-binary configurable API server. 49 | 50 | Options: 51 | `) 52 | flagset.PrintDefaults() 53 | fmt.Fprintf(os.Stderr, ` 54 | (c) RapidLoop, Inc. 2022 * https://rapidrows.io 55 | `) 56 | } 57 | 58 | func main() { 59 | flagset.Usage = usage 60 | if err := flagset.Parse(os.Args[1:]); err == pflag.ErrHelp { 61 | return 62 | } else if err != nil || (!*fversion && flagset.NArg() != 1) || (*flog != "text" && *flog != "json") { 63 | usage() 64 | os.Exit(1) 65 | } 66 | 67 | log.SetFlags(0) 68 | if *fversion { 69 | fmt.Printf("rapidrows v%s\n(c) RapidLoop, Inc. 2022 * https://rapidrows.io\n", 70 | version) 71 | return 72 | } 73 | os.Exit(realmain()) 74 | } 75 | 76 | func realmain() int { 77 | // read input file & validate 78 | raw, err := os.ReadFile(flagset.Arg(0)) 79 | if err != nil { 80 | log.Printf("rapidrows: failed to read input: %v", err) 81 | return 1 82 | } 83 | var config rapidrows.APIServerConfig 84 | if *fyaml { 85 | if err := yaml.Unmarshal(raw, &config); err != nil { 86 | log.Printf("rapidrows: failed to decode yaml: %v", err) 87 | return 1 88 | } 89 | } else { 90 | if err := json.Unmarshal(raw, &config); err != nil { 91 | log.Printf("rapidrows: failed to decode json: %v", err) 92 | return 1 93 | } 94 | } 95 | 96 | if *fcheck { // if only check was requested, check, print and exit 97 | var w, e int 98 | for _, r := range config.Validate() { 99 | if r.Warn { 100 | fmt.Print("warning: ") 101 | w++ 102 | } else { 103 | fmt.Print("error: ") 104 | e++ 105 | } 106 | fmt.Println(r.Message) 107 | } 108 | if w > 0 || e > 0 { 109 | fmt.Printf("\n%s: %d error(s), %d warning(s)\n", flagset.Arg(0), e, w) 110 | } 111 | if e > 0 { 112 | return 2 113 | } 114 | return 0 115 | } 116 | 117 | // start the server 118 | zerolog.SetGlobalLevel(zerolog.DebugLevel) 119 | zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs 120 | var logger zerolog.Logger 121 | if *flog == "json" { 122 | logger = zerolog.New(os.Stdout).With().Timestamp().Logger() 123 | } else { 124 | out := zerolog.ConsoleWriter{ 125 | Out: os.Stdout, 126 | TimeFormat: "2006-01-02 15:04:05.999", 127 | NoColor: !isatty.IsTerminal(os.Stdout.Fd()) || *fnocolor, 128 | } 129 | logger = zerolog.New(out).With().Timestamp().Logger() 130 | } 131 | rti := rapidrows.RuntimeInterface{ 132 | Logger: &logger, 133 | CacheSet: cacheSet, 134 | CacheGet: cacheGet, 135 | } 136 | server, err := rapidrows.NewAPIServer(&config, &rti) 137 | if err != nil { 138 | log.Printf("rapidrows: failed to create server: %v", err) 139 | return 1 140 | } 141 | if err := server.Start(); err != nil { 142 | log.Printf("rapidrows: failed to start server: %v", err) 143 | return 1 144 | } 145 | 146 | // wait for ^C 147 | ch := make(chan os.Signal, 1) 148 | signal.Notify(ch, os.Interrupt) 149 | <-ch 150 | signal.Stop(ch) 151 | close(ch) 152 | 153 | // stop the server 154 | if err := server.Stop(time.Minute); err != nil { 155 | log.Printf("rapidrows: warning: failed to stop server: %v", err) 156 | } 157 | 158 | return 0 159 | } 160 | 161 | var cache sync.Map 162 | 163 | func cacheSet(key uint64, value []byte) { 164 | if len(value) == 0 { 165 | cache.Delete(key) 166 | } else { 167 | cache.Store(key, value) 168 | } 169 | } 170 | 171 | func cacheGet(key uint64) (value []byte, found bool) { 172 | if v, ok := cache.Load(key); ok && v != nil { 173 | return v.([]byte), true 174 | } 175 | return nil, false 176 | } 177 | -------------------------------------------------------------------------------- /datasources.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "math" 23 | "net/url" 24 | "strconv" 25 | "strings" 26 | "sync" 27 | "time" 28 | 29 | "github.com/jackc/pgconn" 30 | "github.com/jackc/pgx/v4" 31 | "github.com/jackc/pgx/v4/pgxpool" 32 | "github.com/rs/zerolog" 33 | ) 34 | 35 | type datasources struct { 36 | logger zerolog.Logger 37 | pools sync.Map 38 | timeouts sync.Map 39 | bgctx context.Context 40 | } 41 | 42 | func (d *datasources) start(bgctx context.Context, sources []Datasource) error { 43 | // store bgctx for use as parent of background contexts we create 44 | d.bgctx = bgctx 45 | 46 | // connect to each source 47 | for i := range sources { 48 | s := &sources[i] 49 | pool, err := dsconnect(bgctx, s) 50 | if err != nil { 51 | d.logger.Error().Str("datasource", s.Name).Err(err).Msg("failed to connect to datasource") 52 | d.stop() 53 | return err 54 | } else { 55 | d.logger.Info().Str("datasource", s.Name).Msg("successfully connected to datasource") 56 | d.pools.Store(s.Name, pool) 57 | if s.Timeout != nil && *s.Timeout > 0 { 58 | d.timeouts.Store(s.Name, time.Duration(*s.Timeout*float64(time.Second))) 59 | } 60 | } 61 | } 62 | return nil 63 | } 64 | 65 | func dsconnect(ctx context.Context, s *Datasource) (pool *pgxpool.Pool, err error) { 66 | // create config 67 | cfg, err := ds2cfg(s) 68 | if err != nil { 69 | return 70 | } 71 | 72 | // create context 73 | if s.Timeout != nil && *s.Timeout > 0 { 74 | var cancel context.CancelFunc 75 | ctx, cancel = context.WithTimeout(ctx, time.Duration(*s.Timeout*float64(time.Second))) 76 | defer cancel() 77 | } 78 | 79 | // connect 80 | pool, err = pgxpool.ConnectConfig(ctx, cfg) 81 | return 82 | } 83 | 84 | func ds2cfg(s *Datasource) (*pgxpool.Config, error) { 85 | // regular params 86 | cfg, err := pgxpool.ParseConfig(ds2url(s)) 87 | if err != nil { 88 | return nil, err 89 | } 90 | 91 | // simple protocol 92 | if s.PreferSimpleProtocol { 93 | cfg.ConnConfig.PreferSimpleProtocol = true 94 | } 95 | 96 | // pool params 97 | if p := s.Pool; p != nil { 98 | if p.MinConns != nil && *p.MinConns > 0 && *p.MinConns <= math.MaxInt32 { 99 | cfg.MinConns = int32(*p.MinConns) 100 | } 101 | if p.MaxConns != nil && *p.MaxConns > 0 && *p.MaxConns <= math.MaxInt32 { 102 | cfg.MaxConns = int32(*p.MaxConns) 103 | } 104 | if p.MaxIdleTime != nil && *p.MaxIdleTime > 0 { 105 | cfg.MaxConnIdleTime = time.Duration(*p.MaxIdleTime * float64(time.Second)) 106 | } 107 | if p.MaxConnectedTime != nil && *p.MaxConnectedTime > 0 { 108 | cfg.MaxConnLifetime = time.Duration(*p.MaxConnectedTime * float64(time.Second)) 109 | } 110 | if p.Lazy { 111 | cfg.LazyConnect = true 112 | } 113 | } 114 | 115 | // role 116 | if len(s.Role) > 0 { 117 | // note: the "SET ROLE" does not take a bind parameter, so $1 type 118 | // arguments cannot be used. However, at this point s.Role is 119 | // guaranteed not to contain any special characters. 120 | cfg.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { 121 | if _, err := conn.Exec(ctx, "SET ROLE "+s.Role); err != nil { 122 | return fmt.Errorf("failed to set role %q: %w", s.Role, err) 123 | } 124 | return nil 125 | } 126 | } 127 | 128 | return cfg, nil 129 | } 130 | 131 | func ds2url(s *Datasource) string { 132 | params := make(url.Values) 133 | set := func(s, kw string) { 134 | if len(s) > 0 { 135 | params.Set(kw, s) 136 | } 137 | } 138 | set(s.Host, "host") // pass as query param, not userinfo 139 | set(s.User, "user") // pass as query param, not userinfo 140 | set(s.Password, "password") // pass as query param, not userinfo 141 | set(s.Database, "dbname") // pass as query param, not userinfo 142 | set(s.Passfile, "passfile") 143 | set(s.SSLMode, "sslmode") 144 | set(s.SSLCert, "sslcert") 145 | set(s.SSLKey, "sslkey") 146 | set(s.SSLRootCert, "sslrootcert") 147 | for k, v := range s.Params { 148 | params.Set(k, v) 149 | } 150 | 151 | // set connection timeout from s.Timeout 152 | if s.Timeout != nil && *s.Timeout > 0 { 153 | params.Set("connect_timeout", strconv.Itoa(int(math.Round(*s.Timeout)))) 154 | } 155 | // note: we're also using context deadline for this instead, only 1 is 156 | // required probably 157 | 158 | return "postgres://?" + params.Encode() 159 | } 160 | 161 | func (d *datasources) get(name string) (*pgxpool.Pool, error) { 162 | v, ok := d.pools.Load(name) 163 | if !ok || v == nil { 164 | return nil, fmt.Errorf("datasource %q not found", name) // should not happen 165 | } 166 | pool, _ := v.(*pgxpool.Pool) 167 | return pool, nil 168 | } 169 | 170 | func (d *datasources) withConn(name string, cb func(conn *pgxpool.Conn) error) error { 171 | // get pool 172 | pool, err := d.get(name) 173 | if err != nil { 174 | return err 175 | } 176 | 177 | // create context 178 | ctx := d.bgctx 179 | if t, ok := d.timeouts.Load(name); ok { 180 | var cancel context.CancelFunc 181 | ctx, cancel = context.WithTimeout(ctx, t.(time.Duration)) 182 | defer cancel() 183 | } 184 | 185 | // acquire conn 186 | return pool.AcquireFunc(ctx, cb) 187 | } 188 | 189 | func (d *datasources) acquire(name string, timeout time.Duration) (*pgxpool.Conn, error) { 190 | // get pool 191 | pool, err := d.get(name) 192 | if err != nil { 193 | return nil, err 194 | } 195 | 196 | // create context 197 | ctx := d.bgctx 198 | if timeout > 0 { 199 | // try to use supplied timeout if valid 200 | var cancel context.CancelFunc 201 | ctx, cancel = context.WithTimeout(ctx, timeout) 202 | defer cancel() 203 | } else if t, ok := d.timeouts.Load(name); ok { 204 | // else use the timeout configured along with the datasource, if present 205 | var cancel context.CancelFunc 206 | ctx, cancel = context.WithTimeout(ctx, t.(time.Duration)) 207 | defer cancel() 208 | } 209 | 210 | // acquire conn 211 | return pool.Acquire(ctx) 212 | } 213 | 214 | func (d *datasources) hijack(name string) (conn *pgx.Conn, err error) { 215 | // get pool 216 | pool, err := d.get(name) 217 | if err != nil { 218 | return nil, err 219 | } 220 | 221 | // acquire one 222 | poolConn, err := pool.Acquire(d.bgctx) 223 | if err != nil { 224 | return 225 | } 226 | 227 | // hijack it 228 | conn = poolConn.Hijack() 229 | return 230 | } 231 | 232 | type querier interface { 233 | Exec(ctx context.Context, sql string, arguments ...interface{}) (commandTag pgconn.CommandTag, err error) 234 | Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) 235 | } 236 | 237 | func (d *datasources) withTx(name string, txopt *TxOptions, cb func(q querier) error) error { 238 | // if tx is nil, reduce this to withConn 239 | if txopt == nil { 240 | adapter1 := func(conn *pgxpool.Conn) error { return cb(conn) } 241 | return d.withConn(name, adapter1) 242 | } 243 | 244 | // get pool 245 | pool, err := d.get(name) 246 | if err != nil { 247 | return err 248 | } 249 | 250 | // create context 251 | ctx := context.Background() 252 | if t, ok := d.timeouts.Load(name); ok { 253 | var cancel context.CancelFunc 254 | ctx, cancel = context.WithTimeout(ctx, t.(time.Duration)) 255 | defer cancel() 256 | } 257 | 258 | // acquire conn and call cb in a tx 259 | opt := pgx.TxOptions{ 260 | AccessMode: pgx.TxAccessMode(strings.ToLower(txopt.Access)), 261 | IsoLevel: pgx.TxIsoLevel(strings.ToLower(txopt.ISOLevel)), 262 | DeferrableMode: pgx.TxDeferrableMode(pick(txopt.Deferrable, "deferrable", "not deferrable")), 263 | } 264 | adapter2 := func(tx pgx.Tx) error { return cb(tx) } 265 | return pool.BeginTxFunc(ctx, opt, adapter2) 266 | } 267 | 268 | func (d *datasources) stop() { 269 | d.pools.Range(func(k, v any) bool { 270 | name, _ := k.(string) 271 | pool, _ := v.(*pgxpool.Pool) 272 | pool.Close() 273 | d.logger.Info().Str("datasource", name).Msg("datasource connection pool closed") 274 | return true 275 | }) 276 | } 277 | -------------------------------------------------------------------------------- /datasources_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows_test 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | "github.com/rapidloop/rapidrows" 24 | "github.com/stretchr/testify/require" 25 | ) 26 | 27 | const cfgTestDatasourcesBasic = `{ 28 | "version": "1", 29 | "listen": "127.0.0.1:60000", 30 | "endpoints": [ 31 | { 32 | "uri": "/", 33 | "implType": "query-json", 34 | "script": "select count(*) from pg_stat_activity where application_name='rrdstest'", 35 | "datasource": "ds1" 36 | }, 37 | { 38 | "uri": "/username", 39 | "implType": "javascript", 40 | "script": "$sys.result = $sys.acquire('ds1').query('select current_user').rows[0][0]", 41 | "datasource": "ds1" 42 | }, 43 | { 44 | "uri": "/setup", 45 | "implType": "exec", 46 | "script": "drop table if exists movies; create table movies (name text, year integer); insert into movies values ('The Shawshank Redemption', 1994), ('The Godfather', 1972), ('The Dark Knight', 2008), ('The Godfather Part II', 1974), ('12 Angry Men', 1957);", 47 | "datasource": "ds1" 48 | }, 49 | { 50 | "uri": "/tx-rw-s", 51 | "implType": "exec", 52 | "tx": { "access": "read write", "level": "serializable" }, 53 | "script": "update movies set name='x' where name='y'", 54 | "datasource": "ds1" 55 | }, 56 | { 57 | "uri": "/tx-ro-s", 58 | "implType": "exec", 59 | "tx": { "access": "read only", "level": "serializable" }, 60 | "script": "update movies set name='x' where name='y'", 61 | "datasource": "ds1" 62 | }, 63 | { 64 | "uri": "/tx-ro-s-d", 65 | "implType": "exec", 66 | "tx": { "access": "read only", "level": "serializable", "deferrable": true }, 67 | "script": "update movies set name='x' where name='y'", 68 | "datasource": "ds1" 69 | }, 70 | { 71 | "uri": "/tx-rw-rr", 72 | "implType": "exec", 73 | "tx": { "level": "repeatable read" }, 74 | "script": "update movies set name='x' where name='y'", 75 | "datasource": "ds1" 76 | } 77 | ], 78 | "datasources": [ 79 | { 80 | "name": "ds1", 81 | "timeout": 10, 82 | "pool": { 83 | "minConns": 5, 84 | "maxConns": 10, 85 | "maxIdleTime": 60, 86 | "maxConnectedTime": 120 87 | }, 88 | "params": { 89 | "application_name": "rrdstest" 90 | } 91 | } 92 | ] 93 | }` 94 | 95 | const expPool1 = `{ 96 | "rows": [ 97 | [ 98 | 5 99 | ] 100 | ] 101 | } 102 | ` 103 | 104 | const expPool2 = `{ 105 | "rows": [ 106 | [ 107 | 1 108 | ] 109 | ] 110 | } 111 | ` 112 | 113 | func TestDatasourcesAcquire(t *testing.T) { 114 | r := require.New(t) 115 | 116 | cfg := loadCfg(r, cfgTestDatasourcesBasic) 117 | s := startServerFull(r, cfg) 118 | body, resp := doGet(r, "http://127.0.0.1:60000/") 119 | r.Equal(200, resp.StatusCode) 120 | r.Equal(expPool1, string(body)) 121 | s.Stop(time.Second) 122 | 123 | cfg.Datasources[0].Pool.Lazy = true 124 | s = startServerFull(r, cfg) 125 | body, resp = doGet(r, "http://127.0.0.1:60000/") 126 | r.Equal(200, resp.StatusCode) 127 | r.Equal(expPool2, string(body)) 128 | 129 | body, resp = doGet(r, "http://127.0.0.1:60000/username") 130 | r.Equal(200, resp.StatusCode) 131 | r.NotEqual("", string(body)) 132 | roleName := string(body) 133 | 134 | s.Stop(time.Second) 135 | 136 | cfg.Datasources[0].Role = roleName 137 | s = startServerFull(r, cfg) 138 | body, resp = doGet(r, "http://127.0.0.1:60000/") 139 | r.Equal(200, resp.StatusCode) 140 | r.Equal(expPool2, string(body)) 141 | s.Stop(time.Second) 142 | 143 | cfg.Datasources[0].Role = "hopefully$no$such$role" 144 | cfg.Datasources[0].Pool.Lazy = false 145 | s, err := rapidrows.NewAPIServer(cfg, nil) 146 | r.NotNil(s, "error was %v", err) 147 | r.Nil(err) 148 | err = s.Start() 149 | r.NotNil(err) 150 | r.EqualError(err, `failed to set role "hopefully$no$such$role": ERROR: role "hopefully$no$such$role" does not exist (SQLSTATE 22023)`) 151 | } 152 | 153 | const expTxRO = `{"rowsAffected":0,"error":"ERROR: cannot execute UPDATE in a read-only transaction (SQLSTATE 25006)"} 154 | ` 155 | 156 | func TestDatasourcesTx(t *testing.T) { 157 | r := require.New(t) 158 | 159 | cfg := loadCfg(r, cfgTestDatasourcesBasic) 160 | s := startServerFull(r, cfg) 161 | 162 | body, resp := doGet(r, "http://127.0.0.1:60000/tx-rw-s") 163 | r.Equal(expMoviesTx1, string(body)) 164 | r.Equal(200, resp.StatusCode) 165 | 166 | body, resp = doGet(r, "http://127.0.0.1:60000/tx-ro-s") 167 | r.Equal(expTxRO, string(body)) 168 | r.Equal(500, resp.StatusCode) 169 | 170 | body, resp = doGet(r, "http://127.0.0.1:60000/tx-ro-s-d") 171 | r.Equal(expTxRO, string(body)) 172 | r.Equal(500, resp.StatusCode) 173 | 174 | body, resp = doGet(r, "http://127.0.0.1:60000/tx-rw-rr") 175 | r.Equal(expMoviesTx1, string(body)) 176 | r.Equal(200, resp.StatusCode) 177 | 178 | s.Stop(time.Second) 179 | } 180 | 181 | /* 182 | 183 | const cfgTestDatasourcesNoConn = `{ 184 | "version": "1", 185 | "listen": "127.0.0.1:60000", 186 | "endpoints": [ 187 | { 188 | "uri": "/", 189 | "implType": "query-json", 190 | "script": "select count(*) from pg_stat_activity where application_name='rrdstest'", 191 | "datasource": "ds1", 192 | "timeout": 1 193 | }, 194 | { 195 | "uri": "/sleep", 196 | "implType": "query-json", 197 | "script": "select pg_sleep(60)", 198 | "datasource": "ds1" 199 | } 200 | ], 201 | "datasources": [ 202 | { 203 | "name": "ds1", 204 | "timeout": 5, 205 | "pool": { 206 | "maxConns": 1 207 | }, 208 | "params": { 209 | "application_name": "rrdstest" 210 | } 211 | } 212 | ] 213 | }` 214 | 215 | func TestDatasourcesNoConn(t *testing.T) { 216 | r := require.New(t) 217 | 218 | cfg := loadCfg(r, cfgTestDatasourcesNoConn) 219 | s := startServerFull(r, cfg) 220 | 221 | go func() { 222 | resp, err := http.Get("http://127.0.0.1:60000/sleep") 223 | r.Nil(err) 224 | r.NotNil(resp) 225 | //r.Equal(200, resp.StatusCode) 226 | r.NotNil(resp.Body) 227 | io.Copy(os.Stdout, resp.Body) 228 | }() 229 | 230 | time.Sleep(time.Second) 231 | body, resp := doGet(r, "http://127.0.0.1:60000/") 232 | fmt.Printf("==%s==\n", string(body)) 233 | r.Equal(500, resp.StatusCode) 234 | 235 | s.Stop(time.Second) 236 | } 237 | */ 238 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // The package rapidrows provides the definition of the API server configuration 18 | // (the [APIServerConfig] structure and it's children), as well as the 19 | // implementation of the API server itself ([APIServer]). Runtime dependencies 20 | // to be supplied by the caller are specified using the [RuntimeInterface]. 21 | // 22 | // Refer to the main RapidRows documentation for more in-depth explanation 23 | // of features as well as examples. The code for the `cmd/rapidrows` CLI tool 24 | // is a good example of how to use the APIServer. 25 | package rapidrows 26 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rapidloop/rapidrows 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/cespare/xxhash/v2 v2.1.2 7 | github.com/go-chi/chi/v5 v5.0.7 8 | github.com/goccy/go-yaml v1.9.5 9 | github.com/jackc/pgconn v1.13.0 10 | github.com/jackc/pgx/v4 v4.17.2 11 | github.com/mattn/go-isatty v0.0.16 12 | github.com/robfig/cron/v3 v3.0.1 13 | github.com/rs/cors v1.8.2 14 | github.com/rs/zerolog v1.28.0 15 | github.com/spf13/pflag v1.0.5 16 | github.com/stretchr/testify v1.8.0 17 | golang.org/x/mod v0.5.1 18 | nhooyr.io/websocket v1.8.7 19 | ) 20 | 21 | require ( 22 | github.com/davecgh/go-spew v1.1.1 // indirect 23 | github.com/fatih/color v1.13.0 // indirect 24 | github.com/jackc/chunkreader/v2 v2.0.1 // indirect 25 | github.com/jackc/pgio v1.0.0 // indirect 26 | github.com/jackc/pgpassfile v1.0.0 // indirect 27 | github.com/jackc/pgproto3/v2 v2.3.1 // indirect 28 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect 29 | github.com/jackc/pgtype v1.12.0 // indirect 30 | github.com/jackc/puddle v1.3.0 // indirect 31 | github.com/klauspost/compress v1.10.3 // indirect 32 | github.com/mattn/go-colorable v0.1.13 // indirect 33 | github.com/pmezard/go-difflib v1.0.0 // indirect 34 | golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect 35 | golang.org/x/sys v0.0.0-20220906165534-d0df966e6959 // indirect 36 | golang.org/x/text v0.3.7 // indirect 37 | golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect 38 | gopkg.in/yaml.v3 v3.0.1 // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /jobs.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows 18 | 19 | import ( 20 | "context" 21 | "fmt" 22 | "time" 23 | 24 | "github.com/robfig/cron/v3" 25 | "github.com/rs/zerolog" 26 | ) 27 | 28 | //------------------------------------------------------------------------------ 29 | // cron 30 | 31 | func newCron(logger zerolog.Logger) *cron.Cron { 32 | l := loggerForCron{logger} 33 | return cron.New(cron.WithLogger(&l)) 34 | } 35 | 36 | type loggerForCron struct { 37 | logger zerolog.Logger 38 | } 39 | 40 | func (l *loggerForCron) Info(msg string, keysAndValues ...interface{}) { 41 | // too verbose 42 | /* 43 | e := l.logger.Info().Bool("crond", true) 44 | for i := 0; i < len(keysAndValues)/2; i += 2 { 45 | e = e.Str(fmt.Sprintf("%v", keysAndValues[i]), fmt.Sprintf("%v", keysAndValues[i+1])) 46 | } 47 | e.Msg(msg) 48 | */ 49 | } 50 | 51 | func (l *loggerForCron) Error(err error, msg string, keysAndValues ...interface{}) { 52 | e := l.logger.Error().Err(err).Bool("crond", true) 53 | for i := 0; i < len(keysAndValues)/2; i += 2 { 54 | e = e.Str(fmt.Sprintf("%v", keysAndValues[i]), fmt.Sprintf("%v", keysAndValues[i+1])) 55 | } 56 | e.Msg(msg) 57 | } 58 | 59 | //------------------------------------------------------------------------------ 60 | // jobs 61 | 62 | func (a *APIServer) setupJobs() error { 63 | // schedule all jobs 64 | for i, job := range a.cfg.Jobs { 65 | if _, err := a.c.AddFunc(job.Schedule, a.jobRunner(i)); err != nil { 66 | a.logger.Error().Err(err).Str("job", job.Name).Msg("failed to schedule job") 67 | return fmt.Errorf("failed to schedule job %q: %v", job.Name, err) 68 | } 69 | } 70 | 71 | return nil 72 | } 73 | 74 | func (a *APIServer) jobRunner(idx int) func() { 75 | return func() { 76 | a.runJob(&a.cfg.Jobs[idx]) 77 | } 78 | } 79 | 80 | func (a *APIServer) runJob(job *Job) { 81 | t0 := time.Now() 82 | logger := a.logger.With().Str("job", job.Name).Logger() 83 | if job.Debug { 84 | logger.Debug().Msg("job starting") 85 | } 86 | 87 | if job.Type == "exec" { 88 | // make context 89 | ctx := a.bgctx 90 | if job.Timeout != nil && *job.Timeout > 0 { 91 | var cancel context.CancelFunc 92 | ctx, cancel = context.WithTimeout(ctx, time.Duration(*job.Timeout*float64(time.Second))) 93 | defer cancel() 94 | } 95 | 96 | // run query 97 | cb := func(q querier) error { 98 | _, err := q.Exec(ctx, job.Script) 99 | return err 100 | } 101 | if err := a.ds.withTx(job.Datasource, job.TxOptions, cb); err != nil { 102 | logger.Error().Err(err).Msg("exec failed") 103 | return 104 | } 105 | } else if job.Type == "javascript" { 106 | if _, _, err := a.runScript(job.Script, make(map[string]any), logger, job.Debug); err != nil { 107 | logger.Error().Err(err).Msg("javascript execution failed") 108 | } 109 | } 110 | 111 | if job.Debug { 112 | logger.Debug().Float64("elapsed", float64(time.Since(t0))/1e6). 113 | Msg("job completed successfully") 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /jobs_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows_test 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | const cfgTestJobsBasic = `{ 27 | "version": "1", 28 | "listen": "127.0.0.1:60000", 29 | "endpoints": [ 30 | { 31 | "uri": "/setup", 32 | "implType": "exec", 33 | "script": "drop table if exists movies; create table movies (name text, year integer); insert into movies values ('The Shawshank Redemption', 1994), ('The Godfather', 1972), ('The Dark Knight', 2008), ('The Godfather Part II', 1974), ('12 Angry Men', 1957);", 34 | "datasource": "default" 35 | } 36 | ], 37 | "jobs": [ 38 | { 39 | "name": "job1", 40 | "type": "exec", 41 | "schedule": "@every 1s", 42 | "datasource": "default", 43 | "script": "update movies set name=name where year=1972", 44 | "timeout": 3, 45 | "debug": true 46 | }, 47 | { 48 | "name": "job2", 49 | "type": "exec", 50 | "schedule": "@every 1s", 51 | "datasource": "default", 52 | "script": "** syntax error", 53 | "debug": true 54 | }, 55 | { 56 | "name": "job3", 57 | "type": "javascript", 58 | "schedule": "@every 1s", 59 | "script": "throw 'foo'" 60 | } 61 | ], 62 | "datasources": [ {"name": "default"} ] 63 | }` 64 | 65 | func TestJobsBasic(t *testing.T) { 66 | r := require.New(t) 67 | 68 | cfg := loadCfg(r, cfgTestJobsBasic) 69 | s := startServerFull(r, cfg) 70 | 71 | time.Sleep(3 * time.Second) 72 | 73 | s.Stop(time.Second) 74 | } 75 | -------------------------------------------------------------------------------- /params.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows 18 | 19 | import ( 20 | "compress/flate" 21 | "compress/gzip" 22 | "encoding/json" 23 | "errors" 24 | "fmt" 25 | "io" 26 | "math" 27 | "net/http" 28 | "net/url" 29 | "regexp" 30 | "strconv" 31 | "strings" 32 | 33 | "github.com/go-chi/chi/v5" 34 | "github.com/rs/zerolog" 35 | ) 36 | 37 | //------------------------------------------------------------------------------ 38 | // parameters 39 | 40 | type paramInfo struct { 41 | rx *regexp.Regexp // compiled "^{.Pattern}$" 42 | enum any // []string, []int64 or []float64 43 | } 44 | 45 | func (a *APIServer) prepareParams() { 46 | for _, ep := range a.cfg.Endpoints { 47 | for _, p := range ep.Params { 48 | var info paramInfo 49 | 50 | // pattern 51 | if len(p.Pattern) > 0 { 52 | if rx, err := regexp.Compile("^" + p.Pattern + "$"); err == nil { 53 | info.rx = rx 54 | } 55 | } 56 | 57 | // enum 58 | if len(p.Enum) > 0 && (p.Type == "string" || p.Type == "integer" || p.Type == "number") { 59 | var sa []string 60 | var ia []int64 61 | var na []float64 62 | for _, v := range p.Enum { 63 | switch p.Type { 64 | case "string": 65 | if s, ok := v.(string); ok { 66 | sa = append(sa, s) 67 | } 68 | case "integer": 69 | if i, ok := v.(int64); ok { 70 | ia = append(ia, i) 71 | } else if i, ok := v.(uint64); ok { 72 | ia = append(ia, int64(i)) // checked to be <=math.MaxInt64 73 | } else if f, ok := v.(float64); ok { 74 | if i, ok := float2int(f); ok { 75 | ia = append(ia, i) 76 | } 77 | } else if s, ok := v.(string); ok { 78 | if i, err := strconv.ParseInt(s, 10, 64); err == nil { 79 | ia = append(ia, i) 80 | } 81 | } 82 | case "number": 83 | if i, ok := v.(int64); ok { 84 | na = append(na, float64(i)) 85 | } else if i, ok := v.(uint64); ok { 86 | na = append(na, float64(i)) 87 | } else if f, ok := v.(float64); ok { 88 | na = append(na, f) 89 | } else if s, ok := v.(string); ok { 90 | if f, err := strconv.ParseFloat(s, 64); err == nil { 91 | na = append(na, f) 92 | } 93 | } 94 | } 95 | } 96 | if len(sa) > 0 { 97 | info.enum = sa 98 | } else if len(ia) > 0 { 99 | info.enum = ia 100 | } else if len(na) > 0 { 101 | info.enum = na 102 | } 103 | } // enum 104 | 105 | if info.rx != nil || info.enum != nil { 106 | a.pinfo.Store(ep.URI+"#"+p.Name, &info) 107 | } 108 | 109 | } // for each param 110 | } // for each endpoint 111 | } 112 | 113 | func (a *APIServer) isSuitable(ep *Endpoint, p *Param, v any) (out any, err error) { 114 | // note: in case of query param or POST form body, v is always a []string 115 | var s string 116 | sv := false 117 | if sa, ok := v.([]string); ok && len(sa) == 1 { 118 | s = sa[0] 119 | sv = true 120 | } else { 121 | s, sv = v.(string) 122 | } 123 | 124 | switch p.Type { 125 | case "string": 126 | if sv { 127 | return a.checkString(ep, p, s) 128 | } 129 | return nil, errors.New("not a string") 130 | case "integer": 131 | if sv { 132 | return a.checkIntegerAny(ep, p, s) 133 | } 134 | return a.checkIntegerAny(ep, p, v) 135 | case "number": 136 | if sv { 137 | return a.checkFloatAny(ep, p, s) 138 | } 139 | return a.checkFloatAny(ep, p, v) 140 | case "boolean": 141 | if sv { 142 | return a.checkBoolAny(ep, p, s) 143 | } 144 | return a.checkBoolAny(ep, p, v) 145 | case "array": 146 | return a.checkArrayAny(ep, p, v) 147 | } 148 | 149 | // should not happen if valid cfg 150 | return nil, errors.New("unknown parameter type") 151 | } 152 | 153 | func (a *APIServer) checkStringAny(ep *Endpoint, p *Param, v any) (string, error) { 154 | if s, ok := v.(string); ok { 155 | return a.checkString(ep, p, s) 156 | } 157 | return "", fmt.Errorf("cannot convert value of type %T to string", v) 158 | } 159 | 160 | func (a *APIServer) checkString(ep *Endpoint, p *Param, s string) (string, error) { 161 | // enum 162 | if len(p.Enum) > 0 { 163 | if pi, ok := a.pinfo.Load(ep.URI + "#" + p.Name); ok && pi != nil { 164 | for _, v := range (pi.(*paramInfo)).enum.([]string) { 165 | if v == s { 166 | return s, nil 167 | } 168 | } 169 | } 170 | return "", errors.New("does not match any of the enumerated values") 171 | } 172 | 173 | // maxLength 174 | if p.MaxLength != nil && *p.MaxLength >= 0 && len(s) > *p.MaxLength { 175 | return "", fmt.Errorf("exceeds specified max length of %d", *p.MaxLength) 176 | } 177 | 178 | // pattern 179 | if len(p.Pattern) > 0 { 180 | if pi, ok := a.pinfo.Load(ep.URI + "#" + p.Name); ok && pi != nil { 181 | if rx := (pi.(*paramInfo)).rx; rx != nil { 182 | if !rx.MatchString(s) { 183 | return "", fmt.Errorf("does not match pattern %s", p.Pattern) 184 | } 185 | } 186 | } 187 | } 188 | 189 | return s, nil 190 | } 191 | 192 | func (a *APIServer) checkIntegerAny(ep *Endpoint, p *Param, v any) (int64, error) { 193 | if s, ok := v.(string); ok { 194 | // allow both "200.00" and "200" 195 | if f, err := strconv.ParseFloat(s, 64); err == nil { 196 | if i, ok := float2int(f); ok { 197 | return a.checkInteger(ep, p, i) 198 | } 199 | } 200 | return 0, errors.New("not a valid integer") 201 | } else if f, ok := v.(float64); ok { 202 | if i, ok := float2int(f); ok { 203 | return a.checkInteger(ep, p, i) 204 | } 205 | } 206 | return 0, fmt.Errorf("cannot convert value of type %T to integer", v) 207 | } 208 | 209 | func (a *APIServer) checkInteger(ep *Endpoint, p *Param, i int64) (int64, error) { 210 | // enum 211 | if len(p.Enum) > 0 { 212 | if pi, ok := a.pinfo.Load(ep.URI + "#" + p.Name); ok && pi != nil { 213 | for _, v := range (pi.(*paramInfo)).enum.([]int64) { 214 | if v == i { 215 | return i, nil 216 | } 217 | } 218 | } 219 | return 0, errors.New("does not match any of the enumerated values") 220 | } 221 | 222 | // minimum 223 | if p.Minimum != nil { 224 | if min := int64(*p.Minimum); i < min { 225 | return 0, fmt.Errorf("is lower than the minimum of %d", min) 226 | } 227 | } 228 | 229 | // maximum 230 | if p.Maximum != nil { 231 | if max := int64(*p.Maximum); i > max { 232 | return 0, fmt.Errorf("is higher than the maximum of %d", max) 233 | } 234 | } 235 | 236 | return i, nil 237 | } 238 | 239 | func (a *APIServer) checkFloatAny(ep *Endpoint, p *Param, v any) (float64, error) { 240 | if s, ok := v.(string); ok { 241 | if f, err := strconv.ParseFloat(s, 64); err != nil { 242 | return 0, errors.New("not a valid number") 243 | } else { 244 | return a.checkFloat(ep, p, f) 245 | } 246 | } else if f, ok := v.(float64); ok && !math.IsNaN(f) && !math.IsInf(f, 0) { 247 | return a.checkFloat(ep, p, f) 248 | } 249 | return 0, fmt.Errorf("cannot convert value of type %T to number", v) 250 | } 251 | 252 | func (a *APIServer) checkFloat(ep *Endpoint, p *Param, f float64) (float64, error) { 253 | // enum 254 | if len(p.Enum) > 0 { 255 | if pi, ok := a.pinfo.Load(ep.URI + "#" + p.Name); ok && pi != nil { 256 | for _, v := range (pi.(*paramInfo)).enum.([]float64) { 257 | if v == f { 258 | return f, nil 259 | } 260 | } 261 | } 262 | return 0, errors.New("does not match any of the enumerated values") 263 | } 264 | 265 | // minimum 266 | if p.Minimum != nil { 267 | if min := float64(*p.Minimum); f < min { 268 | return 0, fmt.Errorf("is lower than the minimum of %g", min) 269 | } 270 | } 271 | 272 | // maximum 273 | if p.Maximum != nil { 274 | if max := float64(*p.Maximum); f > max { 275 | return 0, fmt.Errorf("is higher than the maximum of %g", max) 276 | } 277 | } 278 | 279 | return f, nil 280 | } 281 | 282 | func float2int(f float64) (i int64, ok bool) { 283 | if i, frac := math.Modf(f); math.Abs(frac) < 1e-9 { 284 | return int64(i), true 285 | } 286 | return 0, false 287 | } 288 | 289 | func (a *APIServer) checkBoolAny(ep *Endpoint, p *Param, v any) (out bool, err error) { 290 | if s, ok := v.(string); ok { 291 | s = strings.ToLower(s) 292 | if s == "true" { 293 | return true, nil 294 | } else if s == "false" { 295 | return false, nil 296 | } 297 | } else if b, ok := v.(bool); ok { 298 | return b, nil 299 | } 300 | return false, fmt.Errorf("cannot convert value of type %T to boolean", v) 301 | } 302 | 303 | func (a *APIServer) checkArrayAny(ep *Endpoint, p *Param, v any) (out any, err error) { 304 | if sa, ok := v.([]string); ok { 305 | aa := make([]any, len(sa)) 306 | for i := range sa { 307 | aa[i] = sa[i] 308 | } 309 | return a.checkArray(ep, p, aa) 310 | } else if aa, ok := v.([]any); ok { 311 | return a.checkArray(ep, p, aa) 312 | } 313 | return nil, fmt.Errorf("cannot convert value of type %T to array", v) 314 | } 315 | 316 | func (a *APIServer) checkArray(ep *Endpoint, p *Param, v []any) (out any, err error) { 317 | // minItems 318 | if p.MinItems != nil && len(v) < *p.MinItems { 319 | return nil, fmt.Errorf("fewer than the specified minimum of %d items", *p.MinItems) 320 | } 321 | 322 | // maxItems 323 | if p.MaxItems != nil && len(v) > *p.MaxItems { 324 | return nil, fmt.Errorf("more than the specified maximum of %d items", *p.MaxItems) 325 | } 326 | 327 | // result is one of: 328 | var ( 329 | sa []string 330 | ia []int64 331 | fa []float64 332 | ba []bool 333 | ) 334 | 335 | // for each element: 336 | for j, ev := range v { 337 | switch p.ElemType { 338 | case "integer": 339 | if i, err := a.checkIntegerAny(ep, p, ev); err != nil { 340 | return nil, fmt.Errorf("enum value #%d: %v", j+1, err) 341 | } else { 342 | ia = append(ia, i) 343 | } 344 | case "number": 345 | if f, err := a.checkFloatAny(ep, p, ev); err != nil { 346 | return nil, fmt.Errorf("enum value #%d: %v", j+1, err) 347 | } else { 348 | fa = append(fa, f) 349 | } 350 | case "string": 351 | if s, err := a.checkStringAny(ep, p, ev); err != nil { 352 | return nil, fmt.Errorf("enum value #%d: %v", j+1, err) 353 | } else { 354 | sa = append(sa, s) 355 | } 356 | case "boolean": 357 | if b, err := a.checkBoolAny(ep, p, ev); err != nil { 358 | return nil, fmt.Errorf("enum value #%d: %v", j+1, err) 359 | } else { 360 | ba = append(ba, b) 361 | } 362 | } 363 | } 364 | 365 | // done, return appropriately 366 | switch p.ElemType { 367 | case "integer": 368 | return ia, nil 369 | case "number": 370 | return fa, nil 371 | case "string": 372 | return sa, nil 373 | case "boolean": 374 | return ba, nil 375 | } 376 | // should not happen for valid cfg 377 | return nil, fmt.Errorf("invalid elemType %q", p.ElemType) 378 | } 379 | 380 | func getCT(req *http.Request) (out string) { 381 | out = req.Header.Get("Content-Type") 382 | if pos := strings.IndexByte(out, ';'); pos > 0 { 383 | out = out[:pos] 384 | } 385 | return 386 | } 387 | 388 | func getJSON(req *http.Request, data any) error { 389 | b, err := io.ReadAll(req.Body) 390 | if err != nil { 391 | return err 392 | } 393 | return json.Unmarshal(b, data) 394 | } 395 | 396 | func (a *APIServer) getParams(req *http.Request, ep *Endpoint, 397 | logger zerolog.Logger) ([]any, error) { 398 | 399 | var ( 400 | jsonData map[string]any 401 | formData url.Values 402 | urlData url.Values 403 | ) 404 | if req.Method == "GET" { 405 | urlData = req.URL.Query() 406 | } else { 407 | var wrapped bool 408 | if ce := req.Header.Get("Content-Encoding"); ce == "gzip" { 409 | if r, err := gzip.NewReader(req.Body); err != nil { 410 | logger.Error().Err(err).Msg("failed to initialize gzip reader") 411 | return nil, fmt.Errorf("failed to initialize gzip reader: %v", err) 412 | } else { 413 | wrapped = true 414 | req.Body = r 415 | } 416 | } else if ce == "deflate" { 417 | wrapped = true 418 | req.Body = flate.NewReader(req.Body) 419 | } 420 | if ct := getCT(req); ct == "application/json" { 421 | if err := getJSON(req, &jsonData); err != nil { 422 | logger.Warn().Err(err).Msg("failed to decode json object in request body") 423 | jsonData = nil 424 | } 425 | } else if ct == "application/x-www-form-urlencoded" { 426 | if err := req.ParseForm(); err != nil { 427 | logger.Warn().Err(err).Msg("failed to parse form data in request body") 428 | } else { 429 | formData = req.PostForm 430 | } 431 | } 432 | if wrapped { 433 | if rc, ok := req.Body.(io.Closer); ok { 434 | if err := rc.Close(); err != nil { 435 | logger.Warn().Err(err).Msg("failed to close gzip/deflate reader") 436 | } 437 | } 438 | } 439 | } 440 | 441 | // discard (the rest of the) body, ignore errors, no need to close req.Body 442 | _, _ = io.CopyN(io.Discard, req.Body, 4096) 443 | 444 | getParam := func(in, key string) (v any, ok bool) { 445 | switch in { 446 | case "path": 447 | v = chi.URLParam(req, key) 448 | ok = v != "" 449 | case "query": 450 | v, ok = urlData[key] 451 | case "body": 452 | if jsonData != nil { 453 | v, ok = jsonData[key] 454 | } else if formData != nil { 455 | v, ok = formData[key] 456 | } 457 | } 458 | return 459 | } 460 | 461 | out := make([]any, len(ep.Params)) 462 | for i := range ep.Params { 463 | p := &ep.Params[i] 464 | v, ok := getParam(p.In, p.Name) 465 | if !ok { 466 | if p.Required { 467 | logger.Error().Str("param", p.Name).Msg("value required but not supplied") 468 | return nil, fmt.Errorf("param %q: value required but not supplied", p.Name) 469 | } else { 470 | out[i] = nil 471 | continue 472 | } 473 | } 474 | // special case: boolean url/form parameters with no value will be considered 475 | // as true 476 | if p.Type == "boolean" && 477 | (p.In == "query" || (p.In == "body" && jsonData == nil && formData != nil)) && 478 | ok { 479 | if sa := v.([]string); len(sa) == 1 && len(sa[0]) == 0 { 480 | v = true 481 | } 482 | } 483 | if v2, err := a.isSuitable(ep, p, v); err != nil { 484 | logger.Error().Str("param", p.Name).Err(err).Msg("invalid value") 485 | return nil, fmt.Errorf("param %q: invalid value: %v", p.Name, err) 486 | } else { 487 | out[i] = v2 488 | } 489 | } 490 | 491 | return out, nil 492 | } 493 | -------------------------------------------------------------------------------- /qjs/cutils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * C utilities 3 | * 4 | * Copyright (c) 2017 Fabrice Bellard 5 | * Copyright (c) 2018 Charlie Gordon 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * in the Software without restriction, including without limitation the rights 10 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | * copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be included in 15 | * all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | * THE SOFTWARE. 24 | */ 25 | #ifndef CUTILS_H 26 | #define CUTILS_H 27 | 28 | #include 29 | #include 30 | 31 | /* set if CPU is big endian */ 32 | #undef WORDS_BIGENDIAN 33 | 34 | #define likely(x) __builtin_expect(!!(x), 1) 35 | #define unlikely(x) __builtin_expect(!!(x), 0) 36 | #define force_inline inline __attribute__((always_inline)) 37 | #define no_inline __attribute__((noinline)) 38 | #define __maybe_unused __attribute__((unused)) 39 | 40 | #define xglue(x, y) x ## y 41 | #define glue(x, y) xglue(x, y) 42 | #define stringify(s) tostring(s) 43 | #define tostring(s) #s 44 | 45 | #ifndef offsetof 46 | #define offsetof(type, field) ((size_t) &((type *)0)->field) 47 | #endif 48 | #ifndef countof 49 | #define countof(x) (sizeof(x) / sizeof((x)[0])) 50 | #endif 51 | 52 | typedef int BOOL; 53 | 54 | #ifndef FALSE 55 | enum { 56 | FALSE = 0, 57 | TRUE = 1, 58 | }; 59 | #endif 60 | 61 | void pstrcpy(char *buf, int buf_size, const char *str); 62 | char *pstrcat(char *buf, int buf_size, const char *s); 63 | int strstart(const char *str, const char *val, const char **ptr); 64 | int has_suffix(const char *str, const char *suffix); 65 | 66 | static inline int max_int(int a, int b) 67 | { 68 | if (a > b) 69 | return a; 70 | else 71 | return b; 72 | } 73 | 74 | static inline int min_int(int a, int b) 75 | { 76 | if (a < b) 77 | return a; 78 | else 79 | return b; 80 | } 81 | 82 | static inline uint32_t max_uint32(uint32_t a, uint32_t b) 83 | { 84 | if (a > b) 85 | return a; 86 | else 87 | return b; 88 | } 89 | 90 | static inline uint32_t min_uint32(uint32_t a, uint32_t b) 91 | { 92 | if (a < b) 93 | return a; 94 | else 95 | return b; 96 | } 97 | 98 | static inline int64_t max_int64(int64_t a, int64_t b) 99 | { 100 | if (a > b) 101 | return a; 102 | else 103 | return b; 104 | } 105 | 106 | static inline int64_t min_int64(int64_t a, int64_t b) 107 | { 108 | if (a < b) 109 | return a; 110 | else 111 | return b; 112 | } 113 | 114 | /* WARNING: undefined if a = 0 */ 115 | static inline int clz32(unsigned int a) 116 | { 117 | return __builtin_clz(a); 118 | } 119 | 120 | /* WARNING: undefined if a = 0 */ 121 | static inline int clz64(uint64_t a) 122 | { 123 | return __builtin_clzll(a); 124 | } 125 | 126 | /* WARNING: undefined if a = 0 */ 127 | static inline int ctz32(unsigned int a) 128 | { 129 | return __builtin_ctz(a); 130 | } 131 | 132 | /* WARNING: undefined if a = 0 */ 133 | static inline int ctz64(uint64_t a) 134 | { 135 | return __builtin_ctzll(a); 136 | } 137 | 138 | struct __attribute__((packed)) packed_u64 { 139 | uint64_t v; 140 | }; 141 | 142 | struct __attribute__((packed)) packed_u32 { 143 | uint32_t v; 144 | }; 145 | 146 | struct __attribute__((packed)) packed_u16 { 147 | uint16_t v; 148 | }; 149 | 150 | static inline uint64_t get_u64(const uint8_t *tab) 151 | { 152 | return ((const struct packed_u64 *)tab)->v; 153 | } 154 | 155 | static inline int64_t get_i64(const uint8_t *tab) 156 | { 157 | return (int64_t)((const struct packed_u64 *)tab)->v; 158 | } 159 | 160 | static inline void put_u64(uint8_t *tab, uint64_t val) 161 | { 162 | ((struct packed_u64 *)tab)->v = val; 163 | } 164 | 165 | static inline uint32_t get_u32(const uint8_t *tab) 166 | { 167 | return ((const struct packed_u32 *)tab)->v; 168 | } 169 | 170 | static inline int32_t get_i32(const uint8_t *tab) 171 | { 172 | return (int32_t)((const struct packed_u32 *)tab)->v; 173 | } 174 | 175 | static inline void put_u32(uint8_t *tab, uint32_t val) 176 | { 177 | ((struct packed_u32 *)tab)->v = val; 178 | } 179 | 180 | static inline uint32_t get_u16(const uint8_t *tab) 181 | { 182 | return ((const struct packed_u16 *)tab)->v; 183 | } 184 | 185 | static inline int32_t get_i16(const uint8_t *tab) 186 | { 187 | return (int16_t)((const struct packed_u16 *)tab)->v; 188 | } 189 | 190 | static inline void put_u16(uint8_t *tab, uint16_t val) 191 | { 192 | ((struct packed_u16 *)tab)->v = val; 193 | } 194 | 195 | static inline uint32_t get_u8(const uint8_t *tab) 196 | { 197 | return *tab; 198 | } 199 | 200 | static inline int32_t get_i8(const uint8_t *tab) 201 | { 202 | return (int8_t)*tab; 203 | } 204 | 205 | static inline void put_u8(uint8_t *tab, uint8_t val) 206 | { 207 | *tab = val; 208 | } 209 | 210 | static inline uint16_t bswap16(uint16_t x) 211 | { 212 | return (x >> 8) | (x << 8); 213 | } 214 | 215 | static inline uint32_t bswap32(uint32_t v) 216 | { 217 | return ((v & 0xff000000) >> 24) | ((v & 0x00ff0000) >> 8) | 218 | ((v & 0x0000ff00) << 8) | ((v & 0x000000ff) << 24); 219 | } 220 | 221 | static inline uint64_t bswap64(uint64_t v) 222 | { 223 | return ((v & ((uint64_t)0xff << (7 * 8))) >> (7 * 8)) | 224 | ((v & ((uint64_t)0xff << (6 * 8))) >> (5 * 8)) | 225 | ((v & ((uint64_t)0xff << (5 * 8))) >> (3 * 8)) | 226 | ((v & ((uint64_t)0xff << (4 * 8))) >> (1 * 8)) | 227 | ((v & ((uint64_t)0xff << (3 * 8))) << (1 * 8)) | 228 | ((v & ((uint64_t)0xff << (2 * 8))) << (3 * 8)) | 229 | ((v & ((uint64_t)0xff << (1 * 8))) << (5 * 8)) | 230 | ((v & ((uint64_t)0xff << (0 * 8))) << (7 * 8)); 231 | } 232 | 233 | /* XXX: should take an extra argument to pass slack information to the caller */ 234 | typedef void *DynBufReallocFunc(void *opaque, void *ptr, size_t size); 235 | 236 | typedef struct DynBuf { 237 | uint8_t *buf; 238 | size_t size; 239 | size_t allocated_size; 240 | BOOL error; /* true if a memory allocation error occurred */ 241 | DynBufReallocFunc *realloc_func; 242 | void *opaque; /* for realloc_func */ 243 | } DynBuf; 244 | 245 | void dbuf_init(DynBuf *s); 246 | void dbuf_init2(DynBuf *s, void *opaque, DynBufReallocFunc *realloc_func); 247 | int dbuf_realloc(DynBuf *s, size_t new_size); 248 | int dbuf_write(DynBuf *s, size_t offset, const uint8_t *data, size_t len); 249 | int dbuf_put(DynBuf *s, const uint8_t *data, size_t len); 250 | int dbuf_put_self(DynBuf *s, size_t offset, size_t len); 251 | int dbuf_putc(DynBuf *s, uint8_t c); 252 | int dbuf_putstr(DynBuf *s, const char *str); 253 | static inline int dbuf_put_u16(DynBuf *s, uint16_t val) 254 | { 255 | return dbuf_put(s, (uint8_t *)&val, 2); 256 | } 257 | static inline int dbuf_put_u32(DynBuf *s, uint32_t val) 258 | { 259 | return dbuf_put(s, (uint8_t *)&val, 4); 260 | } 261 | static inline int dbuf_put_u64(DynBuf *s, uint64_t val) 262 | { 263 | return dbuf_put(s, (uint8_t *)&val, 8); 264 | } 265 | int __attribute__((format(printf, 2, 3))) dbuf_printf(DynBuf *s, 266 | const char *fmt, ...); 267 | void dbuf_free(DynBuf *s); 268 | static inline BOOL dbuf_error(DynBuf *s) { 269 | return s->error; 270 | } 271 | static inline void dbuf_set_error(DynBuf *s) 272 | { 273 | s->error = TRUE; 274 | } 275 | 276 | #define UTF8_CHAR_LEN_MAX 6 277 | 278 | int unicode_to_utf8(uint8_t *buf, unsigned int c); 279 | int unicode_from_utf8(const uint8_t *p, int max_len, const uint8_t **pp); 280 | 281 | static inline int from_hex(int c) 282 | { 283 | if (c >= '0' && c <= '9') 284 | return c - '0'; 285 | else if (c >= 'A' && c <= 'F') 286 | return c - 'A' + 10; 287 | else if (c >= 'a' && c <= 'f') 288 | return c - 'a' + 10; 289 | else 290 | return -1; 291 | } 292 | 293 | void rqsort(void *base, size_t nmemb, size_t size, 294 | int (*cmp)(const void *, const void *, void *), 295 | void *arg); 296 | 297 | #endif /* CUTILS_H */ 298 | -------------------------------------------------------------------------------- /qjs/libbf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Tiny arbitrary precision floating point library 3 | * 4 | * Copyright (c) 2017-2021 Fabrice Bellard 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | #ifndef LIBBF_H 25 | #define LIBBF_H 26 | 27 | #include 28 | #include 29 | 30 | #if INTPTR_MAX >= INT64_MAX 31 | #define LIMB_LOG2_BITS 6 32 | #else 33 | #define LIMB_LOG2_BITS 5 34 | #endif 35 | 36 | #define LIMB_BITS (1 << LIMB_LOG2_BITS) 37 | 38 | #if LIMB_BITS == 64 39 | typedef __int128 int128_t; 40 | typedef unsigned __int128 uint128_t; 41 | typedef int64_t slimb_t; 42 | typedef uint64_t limb_t; 43 | typedef uint128_t dlimb_t; 44 | #define BF_RAW_EXP_MIN INT64_MIN 45 | #define BF_RAW_EXP_MAX INT64_MAX 46 | 47 | #define LIMB_DIGITS 19 48 | #define BF_DEC_BASE UINT64_C(10000000000000000000) 49 | 50 | #else 51 | 52 | typedef int32_t slimb_t; 53 | typedef uint32_t limb_t; 54 | typedef uint64_t dlimb_t; 55 | #define BF_RAW_EXP_MIN INT32_MIN 56 | #define BF_RAW_EXP_MAX INT32_MAX 57 | 58 | #define LIMB_DIGITS 9 59 | #define BF_DEC_BASE 1000000000U 60 | 61 | #endif 62 | 63 | /* in bits */ 64 | /* minimum number of bits for the exponent */ 65 | #define BF_EXP_BITS_MIN 3 66 | /* maximum number of bits for the exponent */ 67 | #define BF_EXP_BITS_MAX (LIMB_BITS - 3) 68 | /* extended range for exponent, used internally */ 69 | #define BF_EXT_EXP_BITS_MAX (BF_EXP_BITS_MAX + 1) 70 | /* minimum possible precision */ 71 | #define BF_PREC_MIN 2 72 | /* minimum possible precision */ 73 | #define BF_PREC_MAX (((limb_t)1 << (LIMB_BITS - 2)) - 2) 74 | /* some operations support infinite precision */ 75 | #define BF_PREC_INF (BF_PREC_MAX + 1) /* infinite precision */ 76 | 77 | #if LIMB_BITS == 64 78 | #define BF_CHKSUM_MOD (UINT64_C(975620677) * UINT64_C(9795002197)) 79 | #else 80 | #define BF_CHKSUM_MOD 975620677U 81 | #endif 82 | 83 | #define BF_EXP_ZERO BF_RAW_EXP_MIN 84 | #define BF_EXP_INF (BF_RAW_EXP_MAX - 1) 85 | #define BF_EXP_NAN BF_RAW_EXP_MAX 86 | 87 | /* +/-zero is represented with expn = BF_EXP_ZERO and len = 0, 88 | +/-infinity is represented with expn = BF_EXP_INF and len = 0, 89 | NaN is represented with expn = BF_EXP_NAN and len = 0 (sign is ignored) 90 | */ 91 | typedef struct { 92 | struct bf_context_t *ctx; 93 | int sign; 94 | slimb_t expn; 95 | limb_t len; 96 | limb_t *tab; 97 | } bf_t; 98 | 99 | typedef struct { 100 | /* must be kept identical to bf_t */ 101 | struct bf_context_t *ctx; 102 | int sign; 103 | slimb_t expn; 104 | limb_t len; 105 | limb_t *tab; 106 | } bfdec_t; 107 | 108 | typedef enum { 109 | BF_RNDN, /* round to nearest, ties to even */ 110 | BF_RNDZ, /* round to zero */ 111 | BF_RNDD, /* round to -inf (the code relies on (BF_RNDD xor BF_RNDU) = 1) */ 112 | BF_RNDU, /* round to +inf */ 113 | BF_RNDNA, /* round to nearest, ties away from zero */ 114 | BF_RNDA, /* round away from zero */ 115 | BF_RNDF, /* faithful rounding (nondeterministic, either RNDD or RNDU, 116 | inexact flag is always set) */ 117 | } bf_rnd_t; 118 | 119 | /* allow subnormal numbers. Only available if the number of exponent 120 | bits is <= BF_EXP_BITS_USER_MAX and prec != BF_PREC_INF. */ 121 | #define BF_FLAG_SUBNORMAL (1 << 3) 122 | /* 'prec' is the precision after the radix point instead of the whole 123 | mantissa. Can only be used with bf_round() and 124 | bfdec_[add|sub|mul|div|sqrt|round](). */ 125 | #define BF_FLAG_RADPNT_PREC (1 << 4) 126 | 127 | #define BF_RND_MASK 0x7 128 | #define BF_EXP_BITS_SHIFT 5 129 | #define BF_EXP_BITS_MASK 0x3f 130 | 131 | /* shortcut for bf_set_exp_bits(BF_EXT_EXP_BITS_MAX) */ 132 | #define BF_FLAG_EXT_EXP (BF_EXP_BITS_MASK << BF_EXP_BITS_SHIFT) 133 | 134 | /* contains the rounding mode and number of exponents bits */ 135 | typedef uint32_t bf_flags_t; 136 | 137 | typedef void *bf_realloc_func_t(void *opaque, void *ptr, size_t size); 138 | 139 | typedef struct { 140 | bf_t val; 141 | limb_t prec; 142 | } BFConstCache; 143 | 144 | typedef struct bf_context_t { 145 | void *realloc_opaque; 146 | bf_realloc_func_t *realloc_func; 147 | BFConstCache log2_cache; 148 | BFConstCache pi_cache; 149 | struct BFNTTState *ntt_state; 150 | } bf_context_t; 151 | 152 | static inline int bf_get_exp_bits(bf_flags_t flags) 153 | { 154 | int e; 155 | e = (flags >> BF_EXP_BITS_SHIFT) & BF_EXP_BITS_MASK; 156 | if (e == BF_EXP_BITS_MASK) 157 | return BF_EXP_BITS_MAX + 1; 158 | else 159 | return BF_EXP_BITS_MAX - e; 160 | } 161 | 162 | static inline bf_flags_t bf_set_exp_bits(int n) 163 | { 164 | return ((BF_EXP_BITS_MAX - n) & BF_EXP_BITS_MASK) << BF_EXP_BITS_SHIFT; 165 | } 166 | 167 | /* returned status */ 168 | #define BF_ST_INVALID_OP (1 << 0) 169 | #define BF_ST_DIVIDE_ZERO (1 << 1) 170 | #define BF_ST_OVERFLOW (1 << 2) 171 | #define BF_ST_UNDERFLOW (1 << 3) 172 | #define BF_ST_INEXACT (1 << 4) 173 | /* indicate that a memory allocation error occured. NaN is returned */ 174 | #define BF_ST_MEM_ERROR (1 << 5) 175 | 176 | #define BF_RADIX_MAX 36 /* maximum radix for bf_atof() and bf_ftoa() */ 177 | 178 | static inline slimb_t bf_max(slimb_t a, slimb_t b) 179 | { 180 | if (a > b) 181 | return a; 182 | else 183 | return b; 184 | } 185 | 186 | static inline slimb_t bf_min(slimb_t a, slimb_t b) 187 | { 188 | if (a < b) 189 | return a; 190 | else 191 | return b; 192 | } 193 | 194 | void bf_context_init(bf_context_t *s, bf_realloc_func_t *realloc_func, 195 | void *realloc_opaque); 196 | void bf_context_end(bf_context_t *s); 197 | /* free memory allocated for the bf cache data */ 198 | void bf_clear_cache(bf_context_t *s); 199 | 200 | static inline void *bf_realloc(bf_context_t *s, void *ptr, size_t size) 201 | { 202 | return s->realloc_func(s->realloc_opaque, ptr, size); 203 | } 204 | 205 | /* 'size' must be != 0 */ 206 | static inline void *bf_malloc(bf_context_t *s, size_t size) 207 | { 208 | return bf_realloc(s, NULL, size); 209 | } 210 | 211 | static inline void bf_free(bf_context_t *s, void *ptr) 212 | { 213 | /* must test ptr otherwise equivalent to malloc(0) */ 214 | if (ptr) 215 | bf_realloc(s, ptr, 0); 216 | } 217 | 218 | void bf_init(bf_context_t *s, bf_t *r); 219 | 220 | static inline void bf_delete(bf_t *r) 221 | { 222 | bf_context_t *s = r->ctx; 223 | /* we accept to delete a zeroed bf_t structure */ 224 | if (s && r->tab) { 225 | bf_realloc(s, r->tab, 0); 226 | } 227 | } 228 | 229 | static inline void bf_neg(bf_t *r) 230 | { 231 | r->sign ^= 1; 232 | } 233 | 234 | static inline int bf_is_finite(const bf_t *a) 235 | { 236 | return (a->expn < BF_EXP_INF); 237 | } 238 | 239 | static inline int bf_is_nan(const bf_t *a) 240 | { 241 | return (a->expn == BF_EXP_NAN); 242 | } 243 | 244 | static inline int bf_is_zero(const bf_t *a) 245 | { 246 | return (a->expn == BF_EXP_ZERO); 247 | } 248 | 249 | static inline void bf_memcpy(bf_t *r, const bf_t *a) 250 | { 251 | *r = *a; 252 | } 253 | 254 | int bf_set_ui(bf_t *r, uint64_t a); 255 | int bf_set_si(bf_t *r, int64_t a); 256 | void bf_set_nan(bf_t *r); 257 | void bf_set_zero(bf_t *r, int is_neg); 258 | void bf_set_inf(bf_t *r, int is_neg); 259 | int bf_set(bf_t *r, const bf_t *a); 260 | void bf_move(bf_t *r, bf_t *a); 261 | int bf_get_float64(const bf_t *a, double *pres, bf_rnd_t rnd_mode); 262 | int bf_set_float64(bf_t *a, double d); 263 | 264 | int bf_cmpu(const bf_t *a, const bf_t *b); 265 | int bf_cmp_full(const bf_t *a, const bf_t *b); 266 | int bf_cmp(const bf_t *a, const bf_t *b); 267 | static inline int bf_cmp_eq(const bf_t *a, const bf_t *b) 268 | { 269 | return bf_cmp(a, b) == 0; 270 | } 271 | 272 | static inline int bf_cmp_le(const bf_t *a, const bf_t *b) 273 | { 274 | return bf_cmp(a, b) <= 0; 275 | } 276 | 277 | static inline int bf_cmp_lt(const bf_t *a, const bf_t *b) 278 | { 279 | return bf_cmp(a, b) < 0; 280 | } 281 | 282 | int bf_add(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, bf_flags_t flags); 283 | int bf_sub(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, bf_flags_t flags); 284 | int bf_add_si(bf_t *r, const bf_t *a, int64_t b1, limb_t prec, bf_flags_t flags); 285 | int bf_mul(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, bf_flags_t flags); 286 | int bf_mul_ui(bf_t *r, const bf_t *a, uint64_t b1, limb_t prec, bf_flags_t flags); 287 | int bf_mul_si(bf_t *r, const bf_t *a, int64_t b1, limb_t prec, 288 | bf_flags_t flags); 289 | int bf_mul_2exp(bf_t *r, slimb_t e, limb_t prec, bf_flags_t flags); 290 | int bf_div(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, bf_flags_t flags); 291 | #define BF_DIVREM_EUCLIDIAN BF_RNDF 292 | int bf_divrem(bf_t *q, bf_t *r, const bf_t *a, const bf_t *b, 293 | limb_t prec, bf_flags_t flags, int rnd_mode); 294 | int bf_rem(bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, 295 | bf_flags_t flags, int rnd_mode); 296 | int bf_remquo(slimb_t *pq, bf_t *r, const bf_t *a, const bf_t *b, limb_t prec, 297 | bf_flags_t flags, int rnd_mode); 298 | /* round to integer with infinite precision */ 299 | int bf_rint(bf_t *r, int rnd_mode); 300 | int bf_round(bf_t *r, limb_t prec, bf_flags_t flags); 301 | int bf_sqrtrem(bf_t *r, bf_t *rem1, const bf_t *a); 302 | int bf_sqrt(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 303 | slimb_t bf_get_exp_min(const bf_t *a); 304 | int bf_logic_or(bf_t *r, const bf_t *a, const bf_t *b); 305 | int bf_logic_xor(bf_t *r, const bf_t *a, const bf_t *b); 306 | int bf_logic_and(bf_t *r, const bf_t *a, const bf_t *b); 307 | 308 | /* additional flags for bf_atof */ 309 | /* do not accept hex radix prefix (0x or 0X) if radix = 0 or radix = 16 */ 310 | #define BF_ATOF_NO_HEX (1 << 16) 311 | /* accept binary (0b or 0B) or octal (0o or 0O) radix prefix if radix = 0 */ 312 | #define BF_ATOF_BIN_OCT (1 << 17) 313 | /* Do not parse NaN or Inf */ 314 | #define BF_ATOF_NO_NAN_INF (1 << 18) 315 | /* return the exponent separately */ 316 | #define BF_ATOF_EXPONENT (1 << 19) 317 | 318 | int bf_atof(bf_t *a, const char *str, const char **pnext, int radix, 319 | limb_t prec, bf_flags_t flags); 320 | /* this version accepts prec = BF_PREC_INF and returns the radix 321 | exponent */ 322 | int bf_atof2(bf_t *r, slimb_t *pexponent, 323 | const char *str, const char **pnext, int radix, 324 | limb_t prec, bf_flags_t flags); 325 | int bf_mul_pow_radix(bf_t *r, const bf_t *T, limb_t radix, 326 | slimb_t expn, limb_t prec, bf_flags_t flags); 327 | 328 | 329 | /* Conversion of floating point number to string. Return a null 330 | terminated string or NULL if memory error. *plen contains its 331 | length if plen != NULL. The exponent letter is "e" for base 10, 332 | "p" for bases 2, 8, 16 with a binary exponent and "@" for the other 333 | bases. */ 334 | 335 | #define BF_FTOA_FORMAT_MASK (3 << 16) 336 | 337 | /* fixed format: prec significant digits rounded with (flags & 338 | BF_RND_MASK). Exponential notation is used if too many zeros are 339 | needed.*/ 340 | #define BF_FTOA_FORMAT_FIXED (0 << 16) 341 | /* fractional format: prec digits after the decimal point rounded with 342 | (flags & BF_RND_MASK) */ 343 | #define BF_FTOA_FORMAT_FRAC (1 << 16) 344 | /* free format: 345 | 346 | For binary radices with bf_ftoa() and for bfdec_ftoa(): use the minimum 347 | number of digits to represent 'a'. The precision and the rounding 348 | mode are ignored. 349 | 350 | For the non binary radices with bf_ftoa(): use as many digits as 351 | necessary so that bf_atof() return the same number when using 352 | precision 'prec', rounding to nearest and the subnormal 353 | configuration of 'flags'. The result is meaningful only if 'a' is 354 | already rounded to 'prec' bits. If the subnormal flag is set, the 355 | exponent in 'flags' must also be set to the desired exponent range. 356 | */ 357 | #define BF_FTOA_FORMAT_FREE (2 << 16) 358 | /* same as BF_FTOA_FORMAT_FREE but uses the minimum number of digits 359 | (takes more computation time). Identical to BF_FTOA_FORMAT_FREE for 360 | binary radices with bf_ftoa() and for bfdec_ftoa(). */ 361 | #define BF_FTOA_FORMAT_FREE_MIN (3 << 16) 362 | 363 | /* force exponential notation for fixed or free format */ 364 | #define BF_FTOA_FORCE_EXP (1 << 20) 365 | /* add 0x prefix for base 16, 0o prefix for base 8 or 0b prefix for 366 | base 2 if non zero value */ 367 | #define BF_FTOA_ADD_PREFIX (1 << 21) 368 | /* return "Infinity" instead of "Inf" and add a "+" for positive 369 | exponents */ 370 | #define BF_FTOA_JS_QUIRKS (1 << 22) 371 | 372 | char *bf_ftoa(size_t *plen, const bf_t *a, int radix, limb_t prec, 373 | bf_flags_t flags); 374 | 375 | /* modulo 2^n instead of saturation. NaN and infinity return 0 */ 376 | #define BF_GET_INT_MOD (1 << 0) 377 | int bf_get_int32(int *pres, const bf_t *a, int flags); 378 | int bf_get_int64(int64_t *pres, const bf_t *a, int flags); 379 | int bf_get_uint64(uint64_t *pres, const bf_t *a); 380 | 381 | /* the following functions are exported for testing only. */ 382 | void mp_print_str(const char *str, const limb_t *tab, limb_t n); 383 | void bf_print_str(const char *str, const bf_t *a); 384 | int bf_resize(bf_t *r, limb_t len); 385 | int bf_get_fft_size(int *pdpl, int *pnb_mods, limb_t len); 386 | int bf_normalize_and_round(bf_t *r, limb_t prec1, bf_flags_t flags); 387 | int bf_can_round(const bf_t *a, slimb_t prec, bf_rnd_t rnd_mode, slimb_t k); 388 | slimb_t bf_mul_log2_radix(slimb_t a1, unsigned int radix, int is_inv, 389 | int is_ceil1); 390 | int mp_mul(bf_context_t *s, limb_t *result, 391 | const limb_t *op1, limb_t op1_size, 392 | const limb_t *op2, limb_t op2_size); 393 | limb_t mp_add(limb_t *res, const limb_t *op1, const limb_t *op2, 394 | limb_t n, limb_t carry); 395 | limb_t mp_add_ui(limb_t *tab, limb_t b, size_t n); 396 | int mp_sqrtrem(bf_context_t *s, limb_t *tabs, limb_t *taba, limb_t n); 397 | int mp_recip(bf_context_t *s, limb_t *tabr, const limb_t *taba, limb_t n); 398 | limb_t bf_isqrt(limb_t a); 399 | 400 | /* transcendental functions */ 401 | int bf_const_log2(bf_t *T, limb_t prec, bf_flags_t flags); 402 | int bf_const_pi(bf_t *T, limb_t prec, bf_flags_t flags); 403 | int bf_exp(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 404 | int bf_log(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 405 | #define BF_POW_JS_QUIRKS (1 << 16) /* (+/-1)^(+/-Inf) = NaN, 1^NaN = NaN */ 406 | int bf_pow(bf_t *r, const bf_t *x, const bf_t *y, limb_t prec, bf_flags_t flags); 407 | int bf_cos(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 408 | int bf_sin(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 409 | int bf_tan(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 410 | int bf_atan(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 411 | int bf_atan2(bf_t *r, const bf_t *y, const bf_t *x, 412 | limb_t prec, bf_flags_t flags); 413 | int bf_asin(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 414 | int bf_acos(bf_t *r, const bf_t *a, limb_t prec, bf_flags_t flags); 415 | 416 | /* decimal floating point */ 417 | 418 | static inline void bfdec_init(bf_context_t *s, bfdec_t *r) 419 | { 420 | bf_init(s, (bf_t *)r); 421 | } 422 | static inline void bfdec_delete(bfdec_t *r) 423 | { 424 | bf_delete((bf_t *)r); 425 | } 426 | 427 | static inline void bfdec_neg(bfdec_t *r) 428 | { 429 | r->sign ^= 1; 430 | } 431 | 432 | static inline int bfdec_is_finite(const bfdec_t *a) 433 | { 434 | return (a->expn < BF_EXP_INF); 435 | } 436 | 437 | static inline int bfdec_is_nan(const bfdec_t *a) 438 | { 439 | return (a->expn == BF_EXP_NAN); 440 | } 441 | 442 | static inline int bfdec_is_zero(const bfdec_t *a) 443 | { 444 | return (a->expn == BF_EXP_ZERO); 445 | } 446 | 447 | static inline void bfdec_memcpy(bfdec_t *r, const bfdec_t *a) 448 | { 449 | bf_memcpy((bf_t *)r, (const bf_t *)a); 450 | } 451 | 452 | int bfdec_set_ui(bfdec_t *r, uint64_t a); 453 | int bfdec_set_si(bfdec_t *r, int64_t a); 454 | 455 | static inline void bfdec_set_nan(bfdec_t *r) 456 | { 457 | bf_set_nan((bf_t *)r); 458 | } 459 | static inline void bfdec_set_zero(bfdec_t *r, int is_neg) 460 | { 461 | bf_set_zero((bf_t *)r, is_neg); 462 | } 463 | static inline void bfdec_set_inf(bfdec_t *r, int is_neg) 464 | { 465 | bf_set_inf((bf_t *)r, is_neg); 466 | } 467 | static inline int bfdec_set(bfdec_t *r, const bfdec_t *a) 468 | { 469 | return bf_set((bf_t *)r, (bf_t *)a); 470 | } 471 | static inline void bfdec_move(bfdec_t *r, bfdec_t *a) 472 | { 473 | bf_move((bf_t *)r, (bf_t *)a); 474 | } 475 | static inline int bfdec_cmpu(const bfdec_t *a, const bfdec_t *b) 476 | { 477 | return bf_cmpu((const bf_t *)a, (const bf_t *)b); 478 | } 479 | static inline int bfdec_cmp_full(const bfdec_t *a, const bfdec_t *b) 480 | { 481 | return bf_cmp_full((const bf_t *)a, (const bf_t *)b); 482 | } 483 | static inline int bfdec_cmp(const bfdec_t *a, const bfdec_t *b) 484 | { 485 | return bf_cmp((const bf_t *)a, (const bf_t *)b); 486 | } 487 | static inline int bfdec_cmp_eq(const bfdec_t *a, const bfdec_t *b) 488 | { 489 | return bfdec_cmp(a, b) == 0; 490 | } 491 | static inline int bfdec_cmp_le(const bfdec_t *a, const bfdec_t *b) 492 | { 493 | return bfdec_cmp(a, b) <= 0; 494 | } 495 | static inline int bfdec_cmp_lt(const bfdec_t *a, const bfdec_t *b) 496 | { 497 | return bfdec_cmp(a, b) < 0; 498 | } 499 | 500 | int bfdec_add(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec, 501 | bf_flags_t flags); 502 | int bfdec_sub(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec, 503 | bf_flags_t flags); 504 | int bfdec_add_si(bfdec_t *r, const bfdec_t *a, int64_t b1, limb_t prec, 505 | bf_flags_t flags); 506 | int bfdec_mul(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec, 507 | bf_flags_t flags); 508 | int bfdec_mul_si(bfdec_t *r, const bfdec_t *a, int64_t b1, limb_t prec, 509 | bf_flags_t flags); 510 | int bfdec_div(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec, 511 | bf_flags_t flags); 512 | int bfdec_divrem(bfdec_t *q, bfdec_t *r, const bfdec_t *a, const bfdec_t *b, 513 | limb_t prec, bf_flags_t flags, int rnd_mode); 514 | int bfdec_rem(bfdec_t *r, const bfdec_t *a, const bfdec_t *b, limb_t prec, 515 | bf_flags_t flags, int rnd_mode); 516 | int bfdec_rint(bfdec_t *r, int rnd_mode); 517 | int bfdec_sqrt(bfdec_t *r, const bfdec_t *a, limb_t prec, bf_flags_t flags); 518 | int bfdec_round(bfdec_t *r, limb_t prec, bf_flags_t flags); 519 | int bfdec_get_int32(int *pres, const bfdec_t *a); 520 | int bfdec_pow_ui(bfdec_t *r, const bfdec_t *a, limb_t b); 521 | 522 | char *bfdec_ftoa(size_t *plen, const bfdec_t *a, limb_t prec, bf_flags_t flags); 523 | int bfdec_atof(bfdec_t *r, const char *str, const char **pnext, 524 | limb_t prec, bf_flags_t flags); 525 | 526 | /* the following functions are exported for testing only. */ 527 | extern const limb_t mp_pow_dec[LIMB_DIGITS + 1]; 528 | void bfdec_print_str(const char *str, const bfdec_t *a); 529 | static inline int bfdec_resize(bfdec_t *r, limb_t len) 530 | { 531 | return bf_resize((bf_t *)r, len); 532 | } 533 | int bfdec_normalize_and_round(bfdec_t *r, limb_t prec1, bf_flags_t flags); 534 | 535 | #endif /* LIBBF_H */ 536 | -------------------------------------------------------------------------------- /qjs/libregexp-opcode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Regular Expression Engine 3 | * 4 | * Copyright (c) 2017-2018 Fabrice Bellard 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | 25 | #ifdef DEF 26 | 27 | DEF(invalid, 1) /* never used */ 28 | DEF(char, 3) 29 | DEF(char32, 5) 30 | DEF(dot, 1) 31 | DEF(any, 1) /* same as dot but match any character including line terminator */ 32 | DEF(line_start, 1) 33 | DEF(line_end, 1) 34 | DEF(goto, 5) 35 | DEF(split_goto_first, 5) 36 | DEF(split_next_first, 5) 37 | DEF(match, 1) 38 | DEF(save_start, 2) /* save start position */ 39 | DEF(save_end, 2) /* save end position, must come after saved_start */ 40 | DEF(save_reset, 3) /* reset save positions */ 41 | DEF(loop, 5) /* decrement the top the stack and goto if != 0 */ 42 | DEF(push_i32, 5) /* push integer on the stack */ 43 | DEF(drop, 1) 44 | DEF(word_boundary, 1) 45 | DEF(not_word_boundary, 1) 46 | DEF(back_reference, 2) 47 | DEF(backward_back_reference, 2) /* must come after back_reference */ 48 | DEF(range, 3) /* variable length */ 49 | DEF(range32, 3) /* variable length */ 50 | DEF(lookahead, 5) 51 | DEF(negative_lookahead, 5) 52 | DEF(push_char_pos, 1) /* push the character position on the stack */ 53 | DEF(bne_char_pos, 5) /* pop one stack element and jump if equal to the character 54 | position */ 55 | DEF(prev, 1) /* go to the previous char */ 56 | DEF(simple_greedy_quant, 17) 57 | 58 | #endif /* DEF */ 59 | -------------------------------------------------------------------------------- /qjs/libregexp.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Regular Expression Engine 3 | * 4 | * Copyright (c) 2017-2018 Fabrice Bellard 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | #ifndef LIBREGEXP_H 25 | #define LIBREGEXP_H 26 | 27 | #include 28 | 29 | #include "libunicode.h" 30 | 31 | #define LRE_BOOL int /* for documentation purposes */ 32 | 33 | #define LRE_FLAG_GLOBAL (1 << 0) 34 | #define LRE_FLAG_IGNORECASE (1 << 1) 35 | #define LRE_FLAG_MULTILINE (1 << 2) 36 | #define LRE_FLAG_DOTALL (1 << 3) 37 | #define LRE_FLAG_UTF16 (1 << 4) 38 | #define LRE_FLAG_STICKY (1 << 5) 39 | 40 | #define LRE_FLAG_NAMED_GROUPS (1 << 7) /* named groups are present in the regexp */ 41 | 42 | uint8_t *lre_compile(int *plen, char *error_msg, int error_msg_size, 43 | const char *buf, size_t buf_len, int re_flags, 44 | void *opaque); 45 | int lre_get_capture_count(const uint8_t *bc_buf); 46 | int lre_get_flags(const uint8_t *bc_buf); 47 | const char *lre_get_groupnames(const uint8_t *bc_buf); 48 | int lre_exec(uint8_t **capture, 49 | const uint8_t *bc_buf, const uint8_t *cbuf, int cindex, int clen, 50 | int cbuf_type, void *opaque); 51 | 52 | int lre_parse_escape(const uint8_t **pp, int allow_utf16); 53 | LRE_BOOL lre_is_space(int c); 54 | 55 | /* must be provided by the user */ 56 | LRE_BOOL lre_check_stack_overflow(void *opaque, size_t alloca_size); 57 | void *lre_realloc(void *opaque, void *ptr, size_t size); 58 | 59 | /* JS identifier test */ 60 | extern uint32_t const lre_id_start_table_ascii[4]; 61 | extern uint32_t const lre_id_continue_table_ascii[4]; 62 | 63 | static inline int lre_js_is_ident_first(int c) 64 | { 65 | if ((uint32_t)c < 128) { 66 | return (lre_id_start_table_ascii[c >> 5] >> (c & 31)) & 1; 67 | } else { 68 | #ifdef CONFIG_ALL_UNICODE 69 | return lre_is_id_start(c); 70 | #else 71 | return !lre_is_space(c); 72 | #endif 73 | } 74 | } 75 | 76 | static inline int lre_js_is_ident_next(int c) 77 | { 78 | if ((uint32_t)c < 128) { 79 | return (lre_id_continue_table_ascii[c >> 5] >> (c & 31)) & 1; 80 | } else { 81 | /* ZWNJ and ZWJ are accepted in identifiers */ 82 | #ifdef CONFIG_ALL_UNICODE 83 | return lre_is_id_continue(c) || c == 0x200C || c == 0x200D; 84 | #else 85 | return !lre_is_space(c) || c == 0x200C || c == 0x200D; 86 | #endif 87 | } 88 | } 89 | 90 | #undef LRE_BOOL 91 | 92 | #endif /* LIBREGEXP_H */ 93 | -------------------------------------------------------------------------------- /qjs/libunicode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Unicode utilities 3 | * 4 | * Copyright (c) 2017-2018 Fabrice Bellard 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | #ifndef LIBUNICODE_H 25 | #define LIBUNICODE_H 26 | 27 | #include 28 | 29 | #define LRE_BOOL int /* for documentation purposes */ 30 | 31 | /* define it to include all the unicode tables (40KB larger) */ 32 | #define CONFIG_ALL_UNICODE 33 | 34 | #define LRE_CC_RES_LEN_MAX 3 35 | 36 | typedef enum { 37 | UNICODE_NFC, 38 | UNICODE_NFD, 39 | UNICODE_NFKC, 40 | UNICODE_NFKD, 41 | } UnicodeNormalizationEnum; 42 | 43 | int lre_case_conv(uint32_t *res, uint32_t c, int conv_type); 44 | LRE_BOOL lre_is_cased(uint32_t c); 45 | LRE_BOOL lre_is_case_ignorable(uint32_t c); 46 | 47 | /* char ranges */ 48 | 49 | typedef struct { 50 | int len; /* in points, always even */ 51 | int size; 52 | uint32_t *points; /* points sorted by increasing value */ 53 | void *mem_opaque; 54 | void *(*realloc_func)(void *opaque, void *ptr, size_t size); 55 | } CharRange; 56 | 57 | typedef enum { 58 | CR_OP_UNION, 59 | CR_OP_INTER, 60 | CR_OP_XOR, 61 | } CharRangeOpEnum; 62 | 63 | void cr_init(CharRange *cr, void *mem_opaque, void *(*realloc_func)(void *opaque, void *ptr, size_t size)); 64 | void cr_free(CharRange *cr); 65 | int cr_realloc(CharRange *cr, int size); 66 | int cr_copy(CharRange *cr, const CharRange *cr1); 67 | 68 | static inline int cr_add_point(CharRange *cr, uint32_t v) 69 | { 70 | if (cr->len >= cr->size) { 71 | if (cr_realloc(cr, cr->len + 1)) 72 | return -1; 73 | } 74 | cr->points[cr->len++] = v; 75 | return 0; 76 | } 77 | 78 | static inline int cr_add_interval(CharRange *cr, uint32_t c1, uint32_t c2) 79 | { 80 | if ((cr->len + 2) > cr->size) { 81 | if (cr_realloc(cr, cr->len + 2)) 82 | return -1; 83 | } 84 | cr->points[cr->len++] = c1; 85 | cr->points[cr->len++] = c2; 86 | return 0; 87 | } 88 | 89 | int cr_union1(CharRange *cr, const uint32_t *b_pt, int b_len); 90 | 91 | static inline int cr_union_interval(CharRange *cr, uint32_t c1, uint32_t c2) 92 | { 93 | uint32_t b_pt[2]; 94 | b_pt[0] = c1; 95 | b_pt[1] = c2 + 1; 96 | return cr_union1(cr, b_pt, 2); 97 | } 98 | 99 | int cr_op(CharRange *cr, const uint32_t *a_pt, int a_len, 100 | const uint32_t *b_pt, int b_len, int op); 101 | 102 | int cr_invert(CharRange *cr); 103 | 104 | #ifdef CONFIG_ALL_UNICODE 105 | 106 | LRE_BOOL lre_is_id_start(uint32_t c); 107 | LRE_BOOL lre_is_id_continue(uint32_t c); 108 | 109 | int unicode_normalize(uint32_t **pdst, const uint32_t *src, int src_len, 110 | UnicodeNormalizationEnum n_type, 111 | void *opaque, void *(*realloc_func)(void *opaque, void *ptr, size_t size)); 112 | 113 | /* Unicode character range functions */ 114 | 115 | int unicode_script(CharRange *cr, 116 | const char *script_name, LRE_BOOL is_ext); 117 | int unicode_general_category(CharRange *cr, const char *gc_name); 118 | int unicode_prop(CharRange *cr, const char *prop_name); 119 | 120 | #endif /* CONFIG_ALL_UNICODE */ 121 | 122 | #undef LRE_BOOL 123 | 124 | #endif /* LIBUNICODE_H */ 125 | -------------------------------------------------------------------------------- /qjs/list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Linux klist like system 3 | * 4 | * Copyright (c) 2016-2017 Fabrice Bellard 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining a copy 7 | * of this software and associated documentation files (the "Software"), to deal 8 | * in the Software without restriction, including without limitation the rights 9 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | * copies of the Software, and to permit persons to whom the Software is 11 | * furnished to do so, subject to the following conditions: 12 | * 13 | * The above copyright notice and this permission notice shall be included in 14 | * all copies or substantial portions of the Software. 15 | * 16 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | * THE SOFTWARE. 23 | */ 24 | #ifndef LIST_H 25 | #define LIST_H 26 | 27 | #ifndef NULL 28 | #include 29 | #endif 30 | 31 | struct list_head { 32 | struct list_head *prev; 33 | struct list_head *next; 34 | }; 35 | 36 | #define LIST_HEAD_INIT(el) { &(el), &(el) } 37 | 38 | /* return the pointer of type 'type *' containing 'el' as field 'member' */ 39 | #define list_entry(el, type, member) \ 40 | ((type *)((uint8_t *)(el) - offsetof(type, member))) 41 | 42 | static inline void init_list_head(struct list_head *head) 43 | { 44 | head->prev = head; 45 | head->next = head; 46 | } 47 | 48 | /* insert 'el' between 'prev' and 'next' */ 49 | static inline void __list_add(struct list_head *el, 50 | struct list_head *prev, struct list_head *next) 51 | { 52 | prev->next = el; 53 | el->prev = prev; 54 | el->next = next; 55 | next->prev = el; 56 | } 57 | 58 | /* add 'el' at the head of the list 'head' (= after element head) */ 59 | static inline void list_add(struct list_head *el, struct list_head *head) 60 | { 61 | __list_add(el, head, head->next); 62 | } 63 | 64 | /* add 'el' at the end of the list 'head' (= before element head) */ 65 | static inline void list_add_tail(struct list_head *el, struct list_head *head) 66 | { 67 | __list_add(el, head->prev, head); 68 | } 69 | 70 | static inline void list_del(struct list_head *el) 71 | { 72 | struct list_head *prev, *next; 73 | prev = el->prev; 74 | next = el->next; 75 | prev->next = next; 76 | next->prev = prev; 77 | el->prev = NULL; /* fail safe */ 78 | el->next = NULL; /* fail safe */ 79 | } 80 | 81 | static inline int list_empty(struct list_head *el) 82 | { 83 | return el->next == el; 84 | } 85 | 86 | #define list_for_each(el, head) \ 87 | for(el = (head)->next; el != (head); el = el->next) 88 | 89 | #define list_for_each_safe(el, el1, head) \ 90 | for(el = (head)->next, el1 = el->next; el != (head); \ 91 | el = el1, el1 = el->next) 92 | 93 | #define list_for_each_prev(el, head) \ 94 | for(el = (head)->prev; el != (head); el = el->prev) 95 | 96 | #define list_for_each_prev_safe(el, el1, head) \ 97 | for(el = (head)->prev, el1 = el->prev; el != (head); \ 98 | el = el1, el1 = el->prev) 99 | 100 | #endif /* LIST_H */ 101 | -------------------------------------------------------------------------------- /qjs/quickjs-atom.h: -------------------------------------------------------------------------------- 1 | /* 2 | * QuickJS atom definitions 3 | * 4 | * Copyright (c) 2017-2018 Fabrice Bellard 5 | * Copyright (c) 2017-2018 Charlie Gordon 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * in the Software without restriction, including without limitation the rights 10 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | * copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be included in 15 | * all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | * THE SOFTWARE. 24 | */ 25 | 26 | #ifdef DEF 27 | 28 | /* Note: first atoms are considered as keywords in the parser */ 29 | DEF(null, "null") /* must be first */ 30 | DEF(false, "false") 31 | DEF(true, "true") 32 | DEF(if, "if") 33 | DEF(else, "else") 34 | DEF(return, "return") 35 | DEF(var, "var") 36 | DEF(this, "this") 37 | DEF(delete, "delete") 38 | DEF(void, "void") 39 | DEF(typeof, "typeof") 40 | DEF(new, "new") 41 | DEF(in, "in") 42 | DEF(instanceof, "instanceof") 43 | DEF(do, "do") 44 | DEF(while, "while") 45 | DEF(for, "for") 46 | DEF(break, "break") 47 | DEF(continue, "continue") 48 | DEF(switch, "switch") 49 | DEF(case, "case") 50 | DEF(default, "default") 51 | DEF(throw, "throw") 52 | DEF(try, "try") 53 | DEF(catch, "catch") 54 | DEF(finally, "finally") 55 | DEF(function, "function") 56 | DEF(debugger, "debugger") 57 | DEF(with, "with") 58 | /* FutureReservedWord */ 59 | DEF(class, "class") 60 | DEF(const, "const") 61 | DEF(enum, "enum") 62 | DEF(export, "export") 63 | DEF(extends, "extends") 64 | DEF(import, "import") 65 | DEF(super, "super") 66 | /* FutureReservedWords when parsing strict mode code */ 67 | DEF(implements, "implements") 68 | DEF(interface, "interface") 69 | DEF(let, "let") 70 | DEF(package, "package") 71 | DEF(private, "private") 72 | DEF(protected, "protected") 73 | DEF(public, "public") 74 | DEF(static, "static") 75 | DEF(yield, "yield") 76 | DEF(await, "await") 77 | 78 | /* empty string */ 79 | DEF(empty_string, "") 80 | /* identifiers */ 81 | DEF(length, "length") 82 | DEF(fileName, "fileName") 83 | DEF(lineNumber, "lineNumber") 84 | DEF(message, "message") 85 | DEF(errors, "errors") 86 | DEF(stack, "stack") 87 | DEF(name, "name") 88 | DEF(toString, "toString") 89 | DEF(toLocaleString, "toLocaleString") 90 | DEF(valueOf, "valueOf") 91 | DEF(eval, "eval") 92 | DEF(prototype, "prototype") 93 | DEF(constructor, "constructor") 94 | DEF(configurable, "configurable") 95 | DEF(writable, "writable") 96 | DEF(enumerable, "enumerable") 97 | DEF(value, "value") 98 | DEF(get, "get") 99 | DEF(set, "set") 100 | DEF(of, "of") 101 | DEF(__proto__, "__proto__") 102 | DEF(undefined, "undefined") 103 | DEF(number, "number") 104 | DEF(boolean, "boolean") 105 | DEF(string, "string") 106 | DEF(object, "object") 107 | DEF(symbol, "symbol") 108 | DEF(integer, "integer") 109 | DEF(unknown, "unknown") 110 | DEF(arguments, "arguments") 111 | DEF(callee, "callee") 112 | DEF(caller, "caller") 113 | DEF(_eval_, "") 114 | DEF(_ret_, "") 115 | DEF(_var_, "") 116 | DEF(_arg_var_, "") 117 | DEF(_with_, "") 118 | DEF(lastIndex, "lastIndex") 119 | DEF(target, "target") 120 | DEF(index, "index") 121 | DEF(input, "input") 122 | DEF(defineProperties, "defineProperties") 123 | DEF(apply, "apply") 124 | DEF(join, "join") 125 | DEF(concat, "concat") 126 | DEF(split, "split") 127 | DEF(construct, "construct") 128 | DEF(getPrototypeOf, "getPrototypeOf") 129 | DEF(setPrototypeOf, "setPrototypeOf") 130 | DEF(isExtensible, "isExtensible") 131 | DEF(preventExtensions, "preventExtensions") 132 | DEF(has, "has") 133 | DEF(deleteProperty, "deleteProperty") 134 | DEF(defineProperty, "defineProperty") 135 | DEF(getOwnPropertyDescriptor, "getOwnPropertyDescriptor") 136 | DEF(ownKeys, "ownKeys") 137 | DEF(add, "add") 138 | DEF(done, "done") 139 | DEF(next, "next") 140 | DEF(values, "values") 141 | DEF(source, "source") 142 | DEF(flags, "flags") 143 | DEF(global, "global") 144 | DEF(unicode, "unicode") 145 | DEF(raw, "raw") 146 | DEF(new_target, "new.target") 147 | DEF(this_active_func, "this.active_func") 148 | DEF(home_object, "") 149 | DEF(computed_field, "") 150 | DEF(static_computed_field, "") /* must come after computed_fields */ 151 | DEF(class_fields_init, "") 152 | DEF(brand, "") 153 | DEF(hash_constructor, "#constructor") 154 | DEF(as, "as") 155 | DEF(from, "from") 156 | DEF(meta, "meta") 157 | DEF(_default_, "*default*") 158 | DEF(_star_, "*") 159 | DEF(Module, "Module") 160 | DEF(then, "then") 161 | DEF(resolve, "resolve") 162 | DEF(reject, "reject") 163 | DEF(promise, "promise") 164 | DEF(proxy, "proxy") 165 | DEF(revoke, "revoke") 166 | DEF(async, "async") 167 | DEF(exec, "exec") 168 | DEF(groups, "groups") 169 | DEF(status, "status") 170 | DEF(reason, "reason") 171 | DEF(globalThis, "globalThis") 172 | #ifdef CONFIG_BIGNUM 173 | DEF(bigint, "bigint") 174 | DEF(bigfloat, "bigfloat") 175 | DEF(bigdecimal, "bigdecimal") 176 | DEF(roundingMode, "roundingMode") 177 | DEF(maximumSignificantDigits, "maximumSignificantDigits") 178 | DEF(maximumFractionDigits, "maximumFractionDigits") 179 | #endif 180 | #ifdef CONFIG_ATOMICS 181 | DEF(not_equal, "not-equal") 182 | DEF(timed_out, "timed-out") 183 | DEF(ok, "ok") 184 | #endif 185 | DEF(toJSON, "toJSON") 186 | /* class names */ 187 | DEF(Object, "Object") 188 | DEF(Array, "Array") 189 | DEF(Error, "Error") 190 | DEF(Number, "Number") 191 | DEF(String, "String") 192 | DEF(Boolean, "Boolean") 193 | DEF(Symbol, "Symbol") 194 | DEF(Arguments, "Arguments") 195 | DEF(Math, "Math") 196 | DEF(JSON, "JSON") 197 | DEF(Date, "Date") 198 | DEF(Function, "Function") 199 | DEF(GeneratorFunction, "GeneratorFunction") 200 | DEF(ForInIterator, "ForInIterator") 201 | DEF(RegExp, "RegExp") 202 | DEF(ArrayBuffer, "ArrayBuffer") 203 | DEF(SharedArrayBuffer, "SharedArrayBuffer") 204 | /* must keep same order as class IDs for typed arrays */ 205 | DEF(Uint8ClampedArray, "Uint8ClampedArray") 206 | DEF(Int8Array, "Int8Array") 207 | DEF(Uint8Array, "Uint8Array") 208 | DEF(Int16Array, "Int16Array") 209 | DEF(Uint16Array, "Uint16Array") 210 | DEF(Int32Array, "Int32Array") 211 | DEF(Uint32Array, "Uint32Array") 212 | #ifdef CONFIG_BIGNUM 213 | DEF(BigInt64Array, "BigInt64Array") 214 | DEF(BigUint64Array, "BigUint64Array") 215 | #endif 216 | DEF(Float32Array, "Float32Array") 217 | DEF(Float64Array, "Float64Array") 218 | DEF(DataView, "DataView") 219 | #ifdef CONFIG_BIGNUM 220 | DEF(BigInt, "BigInt") 221 | DEF(BigFloat, "BigFloat") 222 | DEF(BigFloatEnv, "BigFloatEnv") 223 | DEF(BigDecimal, "BigDecimal") 224 | DEF(OperatorSet, "OperatorSet") 225 | DEF(Operators, "Operators") 226 | #endif 227 | DEF(Map, "Map") 228 | DEF(Set, "Set") /* Map + 1 */ 229 | DEF(WeakMap, "WeakMap") /* Map + 2 */ 230 | DEF(WeakSet, "WeakSet") /* Map + 3 */ 231 | DEF(Map_Iterator, "Map Iterator") 232 | DEF(Set_Iterator, "Set Iterator") 233 | DEF(Array_Iterator, "Array Iterator") 234 | DEF(String_Iterator, "String Iterator") 235 | DEF(RegExp_String_Iterator, "RegExp String Iterator") 236 | DEF(Generator, "Generator") 237 | DEF(Proxy, "Proxy") 238 | DEF(Promise, "Promise") 239 | DEF(PromiseResolveFunction, "PromiseResolveFunction") 240 | DEF(PromiseRejectFunction, "PromiseRejectFunction") 241 | DEF(AsyncFunction, "AsyncFunction") 242 | DEF(AsyncFunctionResolve, "AsyncFunctionResolve") 243 | DEF(AsyncFunctionReject, "AsyncFunctionReject") 244 | DEF(AsyncGeneratorFunction, "AsyncGeneratorFunction") 245 | DEF(AsyncGenerator, "AsyncGenerator") 246 | DEF(EvalError, "EvalError") 247 | DEF(RangeError, "RangeError") 248 | DEF(ReferenceError, "ReferenceError") 249 | DEF(SyntaxError, "SyntaxError") 250 | DEF(TypeError, "TypeError") 251 | DEF(URIError, "URIError") 252 | DEF(InternalError, "InternalError") 253 | /* private symbols */ 254 | DEF(Private_brand, "") 255 | /* symbols */ 256 | DEF(Symbol_toPrimitive, "Symbol.toPrimitive") 257 | DEF(Symbol_iterator, "Symbol.iterator") 258 | DEF(Symbol_match, "Symbol.match") 259 | DEF(Symbol_matchAll, "Symbol.matchAll") 260 | DEF(Symbol_replace, "Symbol.replace") 261 | DEF(Symbol_search, "Symbol.search") 262 | DEF(Symbol_split, "Symbol.split") 263 | DEF(Symbol_toStringTag, "Symbol.toStringTag") 264 | DEF(Symbol_isConcatSpreadable, "Symbol.isConcatSpreadable") 265 | DEF(Symbol_hasInstance, "Symbol.hasInstance") 266 | DEF(Symbol_species, "Symbol.species") 267 | DEF(Symbol_unscopables, "Symbol.unscopables") 268 | DEF(Symbol_asyncIterator, "Symbol.asyncIterator") 269 | #ifdef CONFIG_BIGNUM 270 | DEF(Symbol_operatorSet, "Symbol.operatorSet") 271 | #endif 272 | 273 | #endif /* DEF */ 274 | -------------------------------------------------------------------------------- /qjs/quickjs-opcode.h: -------------------------------------------------------------------------------- 1 | /* 2 | * QuickJS opcode definitions 3 | * 4 | * Copyright (c) 2017-2018 Fabrice Bellard 5 | * Copyright (c) 2017-2018 Charlie Gordon 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy 8 | * of this software and associated documentation files (the "Software"), to deal 9 | * in the Software without restriction, including without limitation the rights 10 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | * copies of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be included in 15 | * all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | * THE SOFTWARE. 24 | */ 25 | 26 | #ifdef FMT 27 | FMT(none) 28 | FMT(none_int) 29 | FMT(none_loc) 30 | FMT(none_arg) 31 | FMT(none_var_ref) 32 | FMT(u8) 33 | FMT(i8) 34 | FMT(loc8) 35 | FMT(const8) 36 | FMT(label8) 37 | FMT(u16) 38 | FMT(i16) 39 | FMT(label16) 40 | FMT(npop) 41 | FMT(npopx) 42 | FMT(npop_u16) 43 | FMT(loc) 44 | FMT(arg) 45 | FMT(var_ref) 46 | FMT(u32) 47 | FMT(i32) 48 | FMT(const) 49 | FMT(label) 50 | FMT(atom) 51 | FMT(atom_u8) 52 | FMT(atom_u16) 53 | FMT(atom_label_u8) 54 | FMT(atom_label_u16) 55 | FMT(label_u16) 56 | #undef FMT 57 | #endif /* FMT */ 58 | 59 | #ifdef DEF 60 | 61 | #ifndef def 62 | #define def(id, size, n_pop, n_push, f) DEF(id, size, n_pop, n_push, f) 63 | #endif 64 | 65 | DEF(invalid, 1, 0, 0, none) /* never emitted */ 66 | 67 | /* push values */ 68 | DEF( push_i32, 5, 0, 1, i32) 69 | DEF( push_const, 5, 0, 1, const) 70 | DEF( fclosure, 5, 0, 1, const) /* must follow push_const */ 71 | DEF(push_atom_value, 5, 0, 1, atom) 72 | DEF( private_symbol, 5, 0, 1, atom) 73 | DEF( undefined, 1, 0, 1, none) 74 | DEF( null, 1, 0, 1, none) 75 | DEF( push_this, 1, 0, 1, none) /* only used at the start of a function */ 76 | DEF( push_false, 1, 0, 1, none) 77 | DEF( push_true, 1, 0, 1, none) 78 | DEF( object, 1, 0, 1, none) 79 | DEF( special_object, 2, 0, 1, u8) /* only used at the start of a function */ 80 | DEF( rest, 3, 0, 1, u16) /* only used at the start of a function */ 81 | 82 | DEF( drop, 1, 1, 0, none) /* a -> */ 83 | DEF( nip, 1, 2, 1, none) /* a b -> b */ 84 | DEF( nip1, 1, 3, 2, none) /* a b c -> b c */ 85 | DEF( dup, 1, 1, 2, none) /* a -> a a */ 86 | DEF( dup1, 1, 2, 3, none) /* a b -> a a b */ 87 | DEF( dup2, 1, 2, 4, none) /* a b -> a b a b */ 88 | DEF( dup3, 1, 3, 6, none) /* a b c -> a b c a b c */ 89 | DEF( insert2, 1, 2, 3, none) /* obj a -> a obj a (dup_x1) */ 90 | DEF( insert3, 1, 3, 4, none) /* obj prop a -> a obj prop a (dup_x2) */ 91 | DEF( insert4, 1, 4, 5, none) /* this obj prop a -> a this obj prop a */ 92 | DEF( perm3, 1, 3, 3, none) /* obj a b -> a obj b */ 93 | DEF( perm4, 1, 4, 4, none) /* obj prop a b -> a obj prop b */ 94 | DEF( perm5, 1, 5, 5, none) /* this obj prop a b -> a this obj prop b */ 95 | DEF( swap, 1, 2, 2, none) /* a b -> b a */ 96 | DEF( swap2, 1, 4, 4, none) /* a b c d -> c d a b */ 97 | DEF( rot3l, 1, 3, 3, none) /* x a b -> a b x */ 98 | DEF( rot3r, 1, 3, 3, none) /* a b x -> x a b */ 99 | DEF( rot4l, 1, 4, 4, none) /* x a b c -> a b c x */ 100 | DEF( rot5l, 1, 5, 5, none) /* x a b c d -> a b c d x */ 101 | 102 | DEF(call_constructor, 3, 2, 1, npop) /* func new.target args -> ret. arguments are not counted in n_pop */ 103 | DEF( call, 3, 1, 1, npop) /* arguments are not counted in n_pop */ 104 | DEF( tail_call, 3, 1, 0, npop) /* arguments are not counted in n_pop */ 105 | DEF( call_method, 3, 2, 1, npop) /* arguments are not counted in n_pop */ 106 | DEF(tail_call_method, 3, 2, 0, npop) /* arguments are not counted in n_pop */ 107 | DEF( array_from, 3, 0, 1, npop) /* arguments are not counted in n_pop */ 108 | DEF( apply, 3, 3, 1, u16) 109 | DEF( return, 1, 1, 0, none) 110 | DEF( return_undef, 1, 0, 0, none) 111 | DEF(check_ctor_return, 1, 1, 2, none) 112 | DEF( check_ctor, 1, 0, 0, none) 113 | DEF( check_brand, 1, 2, 2, none) /* this_obj func -> this_obj func */ 114 | DEF( add_brand, 1, 2, 0, none) /* this_obj home_obj -> */ 115 | DEF( return_async, 1, 1, 0, none) 116 | DEF( throw, 1, 1, 0, none) 117 | DEF( throw_error, 6, 0, 0, atom_u8) 118 | DEF( eval, 5, 1, 1, npop_u16) /* func args... -> ret_val */ 119 | DEF( apply_eval, 3, 2, 1, u16) /* func array -> ret_eval */ 120 | DEF( regexp, 1, 2, 1, none) /* create a RegExp object from the pattern and a 121 | bytecode string */ 122 | DEF( get_super, 1, 1, 1, none) 123 | DEF( import, 1, 1, 1, none) /* dynamic module import */ 124 | 125 | DEF( check_var, 5, 0, 1, atom) /* check if a variable exists */ 126 | DEF( get_var_undef, 5, 0, 1, atom) /* push undefined if the variable does not exist */ 127 | DEF( get_var, 5, 0, 1, atom) /* throw an exception if the variable does not exist */ 128 | DEF( put_var, 5, 1, 0, atom) /* must come after get_var */ 129 | DEF( put_var_init, 5, 1, 0, atom) /* must come after put_var. Used to initialize a global lexical variable */ 130 | DEF( put_var_strict, 5, 2, 0, atom) /* for strict mode variable write */ 131 | 132 | DEF( get_ref_value, 1, 2, 3, none) 133 | DEF( put_ref_value, 1, 3, 0, none) 134 | 135 | DEF( define_var, 6, 0, 0, atom_u8) 136 | DEF(check_define_var, 6, 0, 0, atom_u8) 137 | DEF( define_func, 6, 1, 0, atom_u8) 138 | DEF( get_field, 5, 1, 1, atom) 139 | DEF( get_field2, 5, 1, 2, atom) 140 | DEF( put_field, 5, 2, 0, atom) 141 | DEF( get_private_field, 1, 2, 1, none) /* obj prop -> value */ 142 | DEF( put_private_field, 1, 3, 0, none) /* obj value prop -> */ 143 | DEF(define_private_field, 1, 3, 1, none) /* obj prop value -> obj */ 144 | DEF( get_array_el, 1, 2, 1, none) 145 | DEF( get_array_el2, 1, 2, 2, none) /* obj prop -> obj value */ 146 | DEF( put_array_el, 1, 3, 0, none) 147 | DEF(get_super_value, 1, 3, 1, none) /* this obj prop -> value */ 148 | DEF(put_super_value, 1, 4, 0, none) /* this obj prop value -> */ 149 | DEF( define_field, 5, 2, 1, atom) 150 | DEF( set_name, 5, 1, 1, atom) 151 | DEF(set_name_computed, 1, 2, 2, none) 152 | DEF( set_proto, 1, 2, 1, none) 153 | DEF(set_home_object, 1, 2, 2, none) 154 | DEF(define_array_el, 1, 3, 2, none) 155 | DEF( append, 1, 3, 2, none) /* append enumerated object, update length */ 156 | DEF(copy_data_properties, 2, 3, 3, u8) 157 | DEF( define_method, 6, 2, 1, atom_u8) 158 | DEF(define_method_computed, 2, 3, 1, u8) /* must come after define_method */ 159 | DEF( define_class, 6, 2, 2, atom_u8) /* parent ctor -> ctor proto */ 160 | DEF( define_class_computed, 6, 3, 3, atom_u8) /* field_name parent ctor -> field_name ctor proto (class with computed name) */ 161 | 162 | DEF( get_loc, 3, 0, 1, loc) 163 | DEF( put_loc, 3, 1, 0, loc) /* must come after get_loc */ 164 | DEF( set_loc, 3, 1, 1, loc) /* must come after put_loc */ 165 | DEF( get_arg, 3, 0, 1, arg) 166 | DEF( put_arg, 3, 1, 0, arg) /* must come after get_arg */ 167 | DEF( set_arg, 3, 1, 1, arg) /* must come after put_arg */ 168 | DEF( get_var_ref, 3, 0, 1, var_ref) 169 | DEF( put_var_ref, 3, 1, 0, var_ref) /* must come after get_var_ref */ 170 | DEF( set_var_ref, 3, 1, 1, var_ref) /* must come after put_var_ref */ 171 | DEF(set_loc_uninitialized, 3, 0, 0, loc) 172 | DEF( get_loc_check, 3, 0, 1, loc) 173 | DEF( put_loc_check, 3, 1, 0, loc) /* must come after get_loc_check */ 174 | DEF( put_loc_check_init, 3, 1, 0, loc) 175 | DEF(get_var_ref_check, 3, 0, 1, var_ref) 176 | DEF(put_var_ref_check, 3, 1, 0, var_ref) /* must come after get_var_ref_check */ 177 | DEF(put_var_ref_check_init, 3, 1, 0, var_ref) 178 | DEF( close_loc, 3, 0, 0, loc) 179 | DEF( if_false, 5, 1, 0, label) 180 | DEF( if_true, 5, 1, 0, label) /* must come after if_false */ 181 | DEF( goto, 5, 0, 0, label) /* must come after if_true */ 182 | DEF( catch, 5, 0, 1, label) 183 | DEF( gosub, 5, 0, 0, label) /* used to execute the finally block */ 184 | DEF( ret, 1, 1, 0, none) /* used to return from the finally block */ 185 | 186 | DEF( to_object, 1, 1, 1, none) 187 | //DEF( to_string, 1, 1, 1, none) 188 | DEF( to_propkey, 1, 1, 1, none) 189 | DEF( to_propkey2, 1, 2, 2, none) 190 | 191 | DEF( with_get_var, 10, 1, 0, atom_label_u8) /* must be in the same order as scope_xxx */ 192 | DEF( with_put_var, 10, 2, 1, atom_label_u8) /* must be in the same order as scope_xxx */ 193 | DEF(with_delete_var, 10, 1, 0, atom_label_u8) /* must be in the same order as scope_xxx */ 194 | DEF( with_make_ref, 10, 1, 0, atom_label_u8) /* must be in the same order as scope_xxx */ 195 | DEF( with_get_ref, 10, 1, 0, atom_label_u8) /* must be in the same order as scope_xxx */ 196 | DEF(with_get_ref_undef, 10, 1, 0, atom_label_u8) 197 | 198 | DEF( make_loc_ref, 7, 0, 2, atom_u16) 199 | DEF( make_arg_ref, 7, 0, 2, atom_u16) 200 | DEF(make_var_ref_ref, 7, 0, 2, atom_u16) 201 | DEF( make_var_ref, 5, 0, 2, atom) 202 | 203 | DEF( for_in_start, 1, 1, 1, none) 204 | DEF( for_of_start, 1, 1, 3, none) 205 | DEF(for_await_of_start, 1, 1, 3, none) 206 | DEF( for_in_next, 1, 1, 3, none) 207 | DEF( for_of_next, 2, 3, 5, u8) 208 | DEF(iterator_check_object, 1, 1, 1, none) 209 | DEF(iterator_get_value_done, 1, 1, 2, none) 210 | DEF( iterator_close, 1, 3, 0, none) 211 | DEF(iterator_close_return, 1, 4, 4, none) 212 | DEF( iterator_next, 1, 4, 4, none) 213 | DEF( iterator_call, 2, 4, 5, u8) 214 | DEF( initial_yield, 1, 0, 0, none) 215 | DEF( yield, 1, 1, 2, none) 216 | DEF( yield_star, 1, 1, 2, none) 217 | DEF(async_yield_star, 1, 1, 2, none) 218 | DEF( await, 1, 1, 1, none) 219 | 220 | /* arithmetic/logic operations */ 221 | DEF( neg, 1, 1, 1, none) 222 | DEF( plus, 1, 1, 1, none) 223 | DEF( dec, 1, 1, 1, none) 224 | DEF( inc, 1, 1, 1, none) 225 | DEF( post_dec, 1, 1, 2, none) 226 | DEF( post_inc, 1, 1, 2, none) 227 | DEF( dec_loc, 2, 0, 0, loc8) 228 | DEF( inc_loc, 2, 0, 0, loc8) 229 | DEF( add_loc, 2, 1, 0, loc8) 230 | DEF( not, 1, 1, 1, none) 231 | DEF( lnot, 1, 1, 1, none) 232 | DEF( typeof, 1, 1, 1, none) 233 | DEF( delete, 1, 2, 1, none) 234 | DEF( delete_var, 5, 0, 1, atom) 235 | 236 | DEF( mul, 1, 2, 1, none) 237 | DEF( div, 1, 2, 1, none) 238 | DEF( mod, 1, 2, 1, none) 239 | DEF( add, 1, 2, 1, none) 240 | DEF( sub, 1, 2, 1, none) 241 | DEF( pow, 1, 2, 1, none) 242 | DEF( shl, 1, 2, 1, none) 243 | DEF( sar, 1, 2, 1, none) 244 | DEF( shr, 1, 2, 1, none) 245 | DEF( lt, 1, 2, 1, none) 246 | DEF( lte, 1, 2, 1, none) 247 | DEF( gt, 1, 2, 1, none) 248 | DEF( gte, 1, 2, 1, none) 249 | DEF( instanceof, 1, 2, 1, none) 250 | DEF( in, 1, 2, 1, none) 251 | DEF( eq, 1, 2, 1, none) 252 | DEF( neq, 1, 2, 1, none) 253 | DEF( strict_eq, 1, 2, 1, none) 254 | DEF( strict_neq, 1, 2, 1, none) 255 | DEF( and, 1, 2, 1, none) 256 | DEF( xor, 1, 2, 1, none) 257 | DEF( or, 1, 2, 1, none) 258 | DEF(is_undefined_or_null, 1, 1, 1, none) 259 | #ifdef CONFIG_BIGNUM 260 | DEF( mul_pow10, 1, 2, 1, none) 261 | DEF( math_mod, 1, 2, 1, none) 262 | #endif 263 | /* must be the last non short and non temporary opcode */ 264 | DEF( nop, 1, 0, 0, none) 265 | 266 | /* temporary opcodes: never emitted in the final bytecode */ 267 | 268 | def( enter_scope, 3, 0, 0, u16) /* emitted in phase 1, removed in phase 2 */ 269 | def( leave_scope, 3, 0, 0, u16) /* emitted in phase 1, removed in phase 2 */ 270 | 271 | def( label, 5, 0, 0, label) /* emitted in phase 1, removed in phase 3 */ 272 | 273 | def(scope_get_var_undef, 7, 0, 1, atom_u16) /* emitted in phase 1, removed in phase 2 */ 274 | def( scope_get_var, 7, 0, 1, atom_u16) /* emitted in phase 1, removed in phase 2 */ 275 | def( scope_put_var, 7, 1, 0, atom_u16) /* emitted in phase 1, removed in phase 2 */ 276 | def(scope_delete_var, 7, 0, 1, atom_u16) /* emitted in phase 1, removed in phase 2 */ 277 | def( scope_make_ref, 11, 0, 2, atom_label_u16) /* emitted in phase 1, removed in phase 2 */ 278 | def( scope_get_ref, 7, 0, 2, atom_u16) /* emitted in phase 1, removed in phase 2 */ 279 | def(scope_put_var_init, 7, 0, 2, atom_u16) /* emitted in phase 1, removed in phase 2 */ 280 | def(scope_get_private_field, 7, 1, 1, atom_u16) /* obj -> value, emitted in phase 1, removed in phase 2 */ 281 | def(scope_get_private_field2, 7, 1, 2, atom_u16) /* obj -> obj value, emitted in phase 1, removed in phase 2 */ 282 | def(scope_put_private_field, 7, 1, 1, atom_u16) /* obj value ->, emitted in phase 1, removed in phase 2 */ 283 | 284 | def( set_class_name, 5, 1, 1, u32) /* emitted in phase 1, removed in phase 2 */ 285 | 286 | def( line_num, 5, 0, 0, u32) /* emitted in phase 1, removed in phase 3 */ 287 | 288 | #if SHORT_OPCODES 289 | DEF( push_minus1, 1, 0, 1, none_int) 290 | DEF( push_0, 1, 0, 1, none_int) 291 | DEF( push_1, 1, 0, 1, none_int) 292 | DEF( push_2, 1, 0, 1, none_int) 293 | DEF( push_3, 1, 0, 1, none_int) 294 | DEF( push_4, 1, 0, 1, none_int) 295 | DEF( push_5, 1, 0, 1, none_int) 296 | DEF( push_6, 1, 0, 1, none_int) 297 | DEF( push_7, 1, 0, 1, none_int) 298 | DEF( push_i8, 2, 0, 1, i8) 299 | DEF( push_i16, 3, 0, 1, i16) 300 | DEF( push_const8, 2, 0, 1, const8) 301 | DEF( fclosure8, 2, 0, 1, const8) /* must follow push_const8 */ 302 | DEF(push_empty_string, 1, 0, 1, none) 303 | 304 | DEF( get_loc8, 2, 0, 1, loc8) 305 | DEF( put_loc8, 2, 1, 0, loc8) 306 | DEF( set_loc8, 2, 1, 1, loc8) 307 | 308 | DEF( get_loc0, 1, 0, 1, none_loc) 309 | DEF( get_loc1, 1, 0, 1, none_loc) 310 | DEF( get_loc2, 1, 0, 1, none_loc) 311 | DEF( get_loc3, 1, 0, 1, none_loc) 312 | DEF( put_loc0, 1, 1, 0, none_loc) 313 | DEF( put_loc1, 1, 1, 0, none_loc) 314 | DEF( put_loc2, 1, 1, 0, none_loc) 315 | DEF( put_loc3, 1, 1, 0, none_loc) 316 | DEF( set_loc0, 1, 1, 1, none_loc) 317 | DEF( set_loc1, 1, 1, 1, none_loc) 318 | DEF( set_loc2, 1, 1, 1, none_loc) 319 | DEF( set_loc3, 1, 1, 1, none_loc) 320 | DEF( get_arg0, 1, 0, 1, none_arg) 321 | DEF( get_arg1, 1, 0, 1, none_arg) 322 | DEF( get_arg2, 1, 0, 1, none_arg) 323 | DEF( get_arg3, 1, 0, 1, none_arg) 324 | DEF( put_arg0, 1, 1, 0, none_arg) 325 | DEF( put_arg1, 1, 1, 0, none_arg) 326 | DEF( put_arg2, 1, 1, 0, none_arg) 327 | DEF( put_arg3, 1, 1, 0, none_arg) 328 | DEF( set_arg0, 1, 1, 1, none_arg) 329 | DEF( set_arg1, 1, 1, 1, none_arg) 330 | DEF( set_arg2, 1, 1, 1, none_arg) 331 | DEF( set_arg3, 1, 1, 1, none_arg) 332 | DEF( get_var_ref0, 1, 0, 1, none_var_ref) 333 | DEF( get_var_ref1, 1, 0, 1, none_var_ref) 334 | DEF( get_var_ref2, 1, 0, 1, none_var_ref) 335 | DEF( get_var_ref3, 1, 0, 1, none_var_ref) 336 | DEF( put_var_ref0, 1, 1, 0, none_var_ref) 337 | DEF( put_var_ref1, 1, 1, 0, none_var_ref) 338 | DEF( put_var_ref2, 1, 1, 0, none_var_ref) 339 | DEF( put_var_ref3, 1, 1, 0, none_var_ref) 340 | DEF( set_var_ref0, 1, 1, 1, none_var_ref) 341 | DEF( set_var_ref1, 1, 1, 1, none_var_ref) 342 | DEF( set_var_ref2, 1, 1, 1, none_var_ref) 343 | DEF( set_var_ref3, 1, 1, 1, none_var_ref) 344 | 345 | DEF( get_length, 1, 1, 1, none) 346 | 347 | DEF( if_false8, 2, 1, 0, label8) 348 | DEF( if_true8, 2, 1, 0, label8) /* must come after if_false8 */ 349 | DEF( goto8, 2, 0, 0, label8) /* must come after if_true8 */ 350 | DEF( goto16, 3, 0, 0, label16) 351 | 352 | DEF( call0, 1, 1, 1, npopx) 353 | DEF( call1, 1, 1, 1, npopx) 354 | DEF( call2, 1, 1, 1, npopx) 355 | DEF( call3, 1, 1, 1, npopx) 356 | 357 | DEF( is_undefined, 1, 1, 1, none) 358 | DEF( is_null, 1, 1, 1, none) 359 | DEF(typeof_is_undefined, 1, 1, 1, none) 360 | DEF( typeof_is_function, 1, 1, 1, none) 361 | #endif 362 | 363 | #undef DEF 364 | #undef def 365 | #endif /* DEF */ 366 | -------------------------------------------------------------------------------- /qjs/wrap.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package qjs 18 | 19 | /* 20 | #cgo CFLAGS: -D_GNU_SOURCE 21 | #cgo CFLAGS: -DCONFIG_BIGNUM 22 | #cgo CFLAGS: -DDUMP_LEAKS 23 | #cgo CFLAGS: -fno-asynchronous-unwind-tables 24 | #cgo LDFLAGS: -lm -lpthread 25 | #include "wrap.h" 26 | */ 27 | import "C" 28 | import ( 29 | "encoding/json" 30 | "errors" 31 | "fmt" 32 | "math/big" 33 | "sync" 34 | "unsafe" 35 | ) 36 | 37 | type Runtime struct { 38 | c *C.JSRuntime 39 | } 40 | 41 | func NewRuntime() *Runtime { 42 | return &Runtime{c: C.JS_NewRuntime()} 43 | } 44 | 45 | func (r *Runtime) Free() { 46 | C.JS_FreeRuntime(r.c) 47 | } 48 | 49 | func (r *Runtime) RunGC() { 50 | C.JS_RunGC(r.c) 51 | } 52 | 53 | type Context struct { 54 | c *C.JSContext 55 | undef Value 56 | funcs []Function 57 | } 58 | 59 | var ctxMap sync.Map // map of *C.JScontext -> *Context 60 | 61 | func (r *Runtime) NewContext() *Context { 62 | ctx := &Context{c: C.JS_NewContext(r.c)} 63 | ctx.undef = newValue(ctx, C.new_undefined()) 64 | ctxMap.Store(uintptr(unsafe.Pointer(ctx.c)), ctx) 65 | return ctx 66 | } 67 | 68 | func (ctx *Context) Free() { 69 | ctxMap.Delete(uintptr(unsafe.Pointer(ctx.c))) 70 | C.JS_FreeContext(ctx.c) 71 | } 72 | 73 | func (ctx *Context) Eval(script string) (result Value, error Value) { 74 | cscript := C.CString(script) 75 | val := C.wrap_eval(ctx.c, cscript, (C.size_t)(len(script))) 76 | C.free(unsafe.Pointer(cscript)) 77 | 78 | if C.JS_IsException(val) == 1 { // exception occurred 79 | e := C.JS_GetException(ctx.c) 80 | return ctx.undef, newValue(ctx, e) 81 | } 82 | 83 | return newValue(ctx, val), ctx.undef 84 | } 85 | 86 | func (ctx *Context) Global() Value { 87 | return newValue(ctx, C.JS_GetGlobalObject(ctx.c)) 88 | } 89 | 90 | func (ctx *Context) Undefined() Value { 91 | return ctx.undef 92 | } 93 | 94 | func (ctx *Context) Object() Value { 95 | return newValue(ctx, C.JS_NewObject(ctx.c)) 96 | } 97 | 98 | func (ctx *Context) Int(i int) Value { 99 | return newValue(ctx, C.JS_NewInt64(ctx.c, C.int64_t(i))) 100 | } 101 | 102 | func (ctx *Context) ObjectViaJSON(v any) (Value, error) { 103 | j, err := json.Marshal(v) 104 | if err != nil { 105 | return ctx.undef, nil 106 | } 107 | js := string(j) 108 | cjs := C.CString(js) 109 | val := C.json_parse(ctx.c, cjs, (C.size_t)(len(js))) 110 | C.free(unsafe.Pointer(cjs)) 111 | if C.JS_IsException(val) == 1 { // exception occurred 112 | return ctx.undef, ex2error(ctx) 113 | } 114 | return newValue(ctx, val), err 115 | } 116 | 117 | // ThrowError is used to do the equivalent of "throw new Error(msg)" from a 118 | // Go function. 119 | func (ctx *Context) ThrowError(msg string) Value { 120 | cmsg := C.CString(msg) 121 | val := C.throw_error(ctx.c, cmsg, (C.size_t)(len(msg))) 122 | C.free(unsafe.Pointer(cmsg)) 123 | return newValue(ctx, val) 124 | } 125 | 126 | type Function func(ctx *Context, this Value, args []Value) Value 127 | 128 | func (ctx *Context) NewFunction(name string, f Function) Value { 129 | ctx.funcs = append(ctx.funcs, f) 130 | idx := len(ctx.funcs) - 1 131 | cname := C.CString(name) 132 | val := C.register_caller(ctx.c, cname, C.int(len(name)), C.int(idx)) 133 | C.free(unsafe.Pointer(cname)) 134 | return newValue(ctx, val) 135 | } 136 | 137 | //export callgo 138 | func callgo(cctx *C.JSContext, this C.JSValue, argc C.int, argv *C.JSValue, magic C.int) C.JSValue { 139 | // get Go context 140 | ctxraw, ok := ctxMap.Load(uintptr(unsafe.Pointer(cctx))) 141 | if !ok || ctxraw == nil { 142 | // panic("unknown C pointer to Go object in ctxMap") 143 | return C.new_undefined() 144 | } 145 | ctx, ok := ctxraw.(*Context) 146 | if !ok || ctx == nil { 147 | // panic("bad Go object in ctxMap") 148 | return C.new_undefined() 149 | } 150 | 151 | // get Go function to call 152 | if int(magic) < 0 || int(magic) >= len(ctx.funcs) { 153 | // panic("bad magic in callback function") 154 | return C.new_undefined() 155 | } 156 | f := ctx.funcs[int(magic)] 157 | 158 | // make args 159 | cargs := unsafe.Slice(argv, int(argc)) 160 | args := make([]Value, 0, len(cargs)) 161 | for _, arg := range cargs { 162 | args = append(args, newValue(ctx, arg)) 163 | } 164 | 165 | // call and get result 166 | result := f(ctx, newValue(ctx, this), args) 167 | return result.c 168 | } 169 | 170 | type Value struct { 171 | ctx *Context 172 | c C.JSValue 173 | } 174 | 175 | func newValue(ctx *Context, c C.JSValue) Value { 176 | return Value{ctx: ctx, c: c} 177 | } 178 | 179 | func (v Value) Free() { 180 | C.JS_FreeValue(v.ctx.c, v.c) 181 | } 182 | 183 | func (v Value) Dup() Value { 184 | return Value{ctx: v.ctx, c: C.JS_DupValue(v.ctx.c, v.c)} 185 | } 186 | 187 | func (v Value) Any() any { 188 | return val2any(v.ctx, v.c) 189 | } 190 | 191 | func (v Value) SetProperty(prop string, value Value) error { 192 | cprop := C.CString(prop) 193 | rc := int(C.JS_SetPropertyStr(v.ctx.c, v.c, cprop, value.c)) 194 | C.free(unsafe.Pointer(cprop)) 195 | if rc == -1 { // exception occurred 196 | return ex2error(v.ctx) 197 | } 198 | return nil 199 | } 200 | 201 | func (v Value) GetProperty(prop string) Value { 202 | cprop := C.CString(prop) 203 | v2 := newValue(v.ctx, C.JS_GetPropertyStr(v.ctx.c, v.c, cprop)) 204 | C.free(unsafe.Pointer(cprop)) 205 | return v2 206 | } 207 | 208 | func (v Value) GetIndex(idx uint32) Value { 209 | return newValue(v.ctx, C.JS_GetPropertyUint32(v.ctx.c, v.c, (C.uint32_t)(idx))) 210 | } 211 | 212 | func (v Value) Tag() int { 213 | return int(C.value_tag(v.c)) 214 | } 215 | 216 | func (v Value) Unmarshal(obj any) error { 217 | if v.Tag() != TagObject { 218 | return errors.New("not an object") 219 | } 220 | var ok C.int64_t 221 | j := C.json_stringify(v.ctx.c, v.c, &ok) 222 | if ok == 0 { 223 | return errors.New("json stringify failed") 224 | } 225 | js := val2str(v.ctx, j) 226 | C.JS_FreeValue(v.ctx.c, j) 227 | return json.Unmarshal([]byte(js), obj) 228 | } 229 | 230 | const ( 231 | TagBigDecimal = -11 // not enabled, qjs extension 232 | TagBigInt = -10 233 | TagBigFloat = -9 // not enabled, qjs extension 234 | TagSymbol = -8 235 | TagString = -7 236 | TagObject = -1 237 | TagInt = 0 238 | TagBool = 1 239 | TagNull = 2 240 | TagUndefined = 3 241 | TagUninitialized = 4 242 | TagCatchOffset = 5 243 | TagException = 6 244 | TagFloat64 = 7 245 | ) 246 | 247 | func val2any(ctx *Context, c C.JSValue) any { 248 | switch tag := int(C.value_tag(c)); tag { 249 | case TagInt: 250 | var val C.int64_t 251 | C.JS_ToInt64(ctx.c, &val, c) 252 | return int64(val) 253 | case TagBool: 254 | val := C.JS_ToBool(ctx.c, c) 255 | return val == 1 256 | case TagFloat64: 257 | var val C.double 258 | C.JS_ToFloat64(ctx.c, &val, c) 259 | return float64(val) 260 | case TagObject: 261 | if C.JS_IsError(ctx.c, c) == 1 { // js Error class 262 | // the error class has a toString method that returns the message 263 | return &Error{Message: val2str(ctx, c)} 264 | } else if C.JS_IsArray(ctx.c, c) == 1 { // js Array class 265 | n := int(C.array_len(ctx.c, c)) 266 | val := make([]any, n) 267 | for i := 0; i < n; i++ { 268 | elem := C.JS_GetPropertyUint32(ctx.c, c, (C.uint32_t)(i)) 269 | val[i] = val2any(ctx, elem) 270 | C.JS_FreeValue(ctx.c, elem) 271 | } 272 | return val 273 | } else { 274 | var ok C.int64_t 275 | j := C.json_stringify(ctx.c, c, &ok) 276 | if ok == 0 { 277 | return errors.New("json stringify failed") 278 | } 279 | js := val2str(ctx, j) 280 | C.JS_FreeValue(ctx.c, j) 281 | var val map[string]any 282 | if err := json.Unmarshal([]byte(js), &val); err != nil { 283 | return fmt.Errorf("json unmarshal failed: %v", err) 284 | } 285 | return val 286 | } 287 | case TagBigInt: 288 | val, ok := new(big.Int).SetString(val2str(ctx, c), 10) 289 | if !ok { 290 | return errors.New("invalid BigInt string") // should not happen 291 | } 292 | if val.IsInt64() { 293 | return val.Int64() 294 | } 295 | return val 296 | case TagString: 297 | return val2str(ctx, c) 298 | case TagSymbol: 299 | // symbol class has a toString method 300 | return val2str(ctx, c) 301 | } 302 | // null, undefined, uninitialized => nil 303 | return nil 304 | } 305 | 306 | func val2str(ctx *Context, v C.JSValue) string { 307 | p := C.JS_ToCString(ctx.c, v) 308 | val := C.GoString(p) 309 | C.JS_FreeCString(ctx.c, p) 310 | return val 311 | } 312 | 313 | // note: frees the qjs exception object! 314 | func ex2error(ctx *Context) error { 315 | e := C.JS_GetException(ctx.c) 316 | msg := val2str(ctx, e) 317 | C.JS_FreeValue(ctx.c, e) 318 | return &Error{Message: msg} 319 | } 320 | 321 | // Error is used only when a Javascript error object has to be returned, 322 | // usually because it was thrown as an exception. 323 | type Error struct { 324 | Message string 325 | } 326 | 327 | func (e Error) Error() string { 328 | return e.Message 329 | } 330 | -------------------------------------------------------------------------------- /qjs/wrap.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #ifndef _WRAP_H_ 18 | #define _WRAP_H_ 19 | 20 | #include // free 21 | #include // int64_t 22 | #include "quickjs.h" 23 | 24 | static inline int64_t value_tag(JSValue v) { 25 | return (int64_t)(JS_VALUE_GET_TAG(v)); 26 | } 27 | 28 | static inline int64_t array_len(JSContext *ctx, JSValue v) { 29 | int64_t out = 0; 30 | JSValue lv = JS_GetPropertyStr(ctx, v, "length"); 31 | if (!JS_IsException(lv)) 32 | JS_ToInt64(ctx, &out, lv); 33 | JS_FreeValue(ctx, lv); 34 | return out; 35 | } 36 | 37 | JSValue callgo(JSContext *ctx, JSValue this_val, int argc, JSValue *argv, int magic); 38 | 39 | static inline JSValue register_caller(JSContext *ctx, const char *name, int length, int magic) { 40 | return JS_NewCFunction2(ctx, (JSCFunction *)callgo, name, length, JS_CFUNC_generic_magic, magic); 41 | } 42 | 43 | static inline JSValue wrap_eval(JSContext *ctx, const char *input, size_t input_len) { 44 | return JS_Eval(ctx, input, input_len, "script", 0); 45 | } 46 | 47 | static inline JSValue json_stringify(JSContext *ctx, JSValueConst obj, int64_t *ok) { 48 | JSValue val = JS_JSONStringify(ctx, obj, JS_UNDEFINED, JS_UNDEFINED); 49 | *ok = (JS_IsString(val) == 1) ? 1 : 0; 50 | if (*ok == 0) { 51 | JS_FreeValue(ctx, val); 52 | val = JS_UNDEFINED; 53 | } 54 | return val; 55 | } 56 | 57 | static inline JSValue new_undefined() { 58 | return JS_UNDEFINED; 59 | } 60 | 61 | static inline JSValue json_parse(JSContext *ctx, const char *input, size_t input_len) { 62 | return JS_ParseJSON(ctx, input, input_len, "object"); 63 | } 64 | 65 | static inline JSValue throw_error(JSContext *ctx, const char *msg, size_t msg_len) { 66 | JSValue err = JS_NewError(ctx); 67 | JSValue msgv = JS_NewStringLen(ctx, msg, msg_len); 68 | int rc = JS_SetPropertyStr(ctx, err, "message", msgv); 69 | if (rc == -1) { // exception occured, ignore 70 | JSValue ex = JS_GetException(ctx); 71 | JS_FreeValue(ctx, ex); 72 | JS_FreeValue(ctx, msgv); 73 | JS_FreeValue(ctx, err); 74 | return JS_UNDEFINED; 75 | } else if (rc == 0) { // failed to set property for some reason (should not happen?) 76 | JS_FreeValue(ctx, msgv); 77 | JS_FreeValue(ctx, err); 78 | return JS_UNDEFINED; 79 | } 80 | 81 | return JS_Throw(ctx, err); // frees err and msgv 82 | } 83 | 84 | #endif // _WRAP_H_ 85 | -------------------------------------------------------------------------------- /script.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows 18 | 19 | import ( 20 | "context" 21 | "encoding/json" 22 | "errors" 23 | "fmt" 24 | "net/http" 25 | "runtime" 26 | "time" 27 | 28 | "github.com/jackc/pgx/v4/pgxpool" 29 | "github.com/rapidloop/rapidrows/qjs" 30 | "github.com/rs/zerolog" 31 | ) 32 | 33 | const jsPropSys = "$sys" 34 | 35 | //------------------------------------------------------------------------------ 36 | 37 | type scriptContext struct { 38 | conns map[string]*pgxpool.Conn 39 | ctx *qjs.Context 40 | a *APIServer 41 | logger zerolog.Logger 42 | debug bool 43 | } 44 | 45 | func newScriptContext(ctx *qjs.Context, a *APIServer, logger zerolog.Logger, 46 | debug bool) *scriptContext { 47 | return &scriptContext{ 48 | ctx: ctx, 49 | conns: make(map[string]*pgxpool.Conn), 50 | a: a, 51 | logger: logger, 52 | debug: debug, 53 | } 54 | } 55 | 56 | func (sctx *scriptContext) acquire(ctx *qjs.Context, this qjs.Value, args []qjs.Value) qjs.Value { 57 | // check args 58 | if n := len(args); n != 1 && n != 2 { 59 | sctx.logger.Error().Msgf("$sys.acquire: got %d args, need 1 or 2", n) 60 | return ctx.ThrowError("$sys.acquire: needs 1 or 2 arguments") 61 | } 62 | 63 | // first arg: required, string, valid data source name 64 | if args[0].Tag() != qjs.TagString { 65 | sctx.logger.Error().Msg("$sys.acquire: first argument not a string") 66 | return ctx.ThrowError("$sys.acquire: first argument must be datasource name (string)") 67 | } 68 | dsname, _ := args[0].Any().(string) 69 | if len(dsname) == 0 { 70 | sctx.logger.Error().Msg("$sys.acquire: first argument is an empty string") 71 | return ctx.ThrowError("$sys.acquire: datasource not specified") 72 | } 73 | found := false 74 | for i := range sctx.a.cfg.Datasources { 75 | if sctx.a.cfg.Datasources[i].Name == dsname { 76 | found = true 77 | break 78 | } 79 | } 80 | if !found { 81 | sctx.logger.Error().Msgf("$sys.acquire: unknown datasource %q", dsname) 82 | return ctx.ThrowError(fmt.Sprintf("$sys.acquire: unknown datasource %q", dsname)) 83 | } 84 | 85 | // second arg: optional, integer, timeout 86 | var timeout time.Duration 87 | if len(args) == 2 { 88 | tag := args[1].Tag() 89 | if tag == qjs.TagInt { 90 | v, _ := args[1].Any().(int64) 91 | timeout = time.Duration(v) * time.Second 92 | } else if tag == qjs.TagFloat64 { 93 | v, _ := args[1].Any().(float64) 94 | timeout = time.Duration(float64(time.Second) * v) 95 | } else { 96 | sctx.logger.Error().Msg("$sys.acquire: second argument not a number") 97 | return ctx.ThrowError("$sys.acquire: second argument must be timeout in seconds (number)") 98 | } 99 | } 100 | 101 | // try to acquire & add to pool 102 | conn, err := sctx.a.ds.acquire(dsname, timeout) 103 | if err != nil { 104 | sctx.logger.Error().Err(err).Str("datasource", dsname). 105 | Msg("$sys.acquire: failed to acquire connection") 106 | return ctx.ThrowError(fmt.Sprintf("$sys.acquire(%q): %v", dsname, err)) 107 | } 108 | sctx.conns[dsname] = conn 109 | 110 | // log if debug 111 | if sctx.debug { 112 | sctx.logger.Debug().Str("datasource", dsname).Msg("acquired connection") 113 | } 114 | 115 | // capture conn 116 | query := func(ctx *qjs.Context, this qjs.Value, args []qjs.Value) qjs.Value { 117 | return sctx.query(conn, ctx, this, args) 118 | } 119 | exec := func(ctx *qjs.Context, this qjs.Value, args []qjs.Value) qjs.Value { 120 | return sctx.exec(conn, ctx, this, args) 121 | } 122 | 123 | // create a javascript object, set methods and return 124 | connObj := ctx.Object() 125 | setfnProp(sctx.ctx, connObj, "query", query) 126 | setfnProp(sctx.ctx, connObj, "exec", exec) 127 | return connObj 128 | } 129 | 130 | func (sctx *scriptContext) query(conn *pgxpool.Conn, ctx *qjs.Context, this qjs.Value, args []qjs.Value) qjs.Value { 131 | // parse args 132 | q, sqlArgs, err := parseArgs("query", args) 133 | if err != nil { 134 | sctx.logger.Error().Err(err).Msg("bad input") 135 | return ctx.ThrowError(err.Error()) 136 | } 137 | 138 | // actually query 139 | t1 := time.Now() 140 | qr := doQuery(sctx.a.bgctx, conn, q, sqlArgs...) 141 | if sctx.debug { 142 | elapsed := float64(time.Since(t1)) / 1e6 143 | if len(qr.Error) == 0 { 144 | sctx.logger.Debug().Float64("elapsed", elapsed).Msg("query completed successfully") 145 | } 146 | } 147 | 148 | // if query failed, throw error 149 | if len(qr.Error) != 0 { 150 | sctx.logger.Error().Str("error", qr.Error).Msg("query failed") 151 | return ctx.ThrowError(qr.Error) 152 | } 153 | 154 | // convert queryResult object to qjs object 155 | ret, err := ctx.ObjectViaJSON(qr) 156 | if err != nil { // should not happen 157 | sctx.logger.Error().Err(err).Msg("json encoding failed") 158 | return ctx.ThrowError(err.Error()) 159 | } 160 | return ret 161 | } 162 | 163 | func (sctx *scriptContext) exec(conn *pgxpool.Conn, ctx *qjs.Context, _ qjs.Value, args []qjs.Value) qjs.Value { 164 | // parse args 165 | q, sqlArgs, err := parseArgs("exec", args) 166 | if err != nil { 167 | sctx.logger.Error().Err(err).Msg("bad input") 168 | return ctx.ThrowError(err.Error()) 169 | } 170 | 171 | // actually exec 172 | t1 := time.Now() 173 | er := doExec(sctx.a.bgctx, conn, q, sqlArgs...) 174 | if sctx.debug { 175 | elapsed := float64(time.Since(t1)) / 1e6 176 | if len(er.Error) == 0 { 177 | sctx.logger.Debug().Float64("elapsed", elapsed).Msg("exec query completed successfully") 178 | } 179 | } 180 | 181 | // if exec query failed, throw error 182 | if len(er.Error) != 0 { 183 | sctx.logger.Error().Str("error", er.Error).Msg("exec query failed") 184 | return ctx.ThrowError(er.Error) 185 | } 186 | 187 | // convert execResult object to qjs object 188 | ret, err := ctx.ObjectViaJSON(er) 189 | if err != nil { // should not happen 190 | sctx.logger.Error().Err(err).Msg("json encoding failed") 191 | return ctx.ThrowError(err.Error()) 192 | } 193 | return ret 194 | } 195 | 196 | func (sctx *scriptContext) close() { 197 | for dsname, conn := range sctx.conns { 198 | conn.Release() 199 | if sctx.debug { 200 | sctx.logger.Debug().Str("datasource", dsname).Msg("released connection") 201 | } 202 | } 203 | } 204 | 205 | func setfnProp(ctx *qjs.Context, obj qjs.Value, name string, f qjs.Function) { 206 | fv := ctx.NewFunction(name, f) 207 | obj.SetProperty(name, fv) 208 | } 209 | 210 | // doQuery runs a sql query with args on a pgxpool connection and collects the 211 | // resultset into a queryResult. If the query failed, the error will be present 212 | // in queryResult.Error. 213 | func doQuery(ctx context.Context, conn *pgxpool.Conn, query string, args ...any) (qr queryResult) { 214 | rows, err := conn.Query(ctx, query, args...) 215 | if err != nil { 216 | qr.Error = err.Error() 217 | return 218 | } 219 | defer rows.Close() 220 | 221 | qr.Rows = make([][]any, 0) 222 | for rows.Next() { 223 | vals, err := rows.Values() 224 | if err != nil { 225 | qr.Rows = nil 226 | qr.Error = err.Error() 227 | return 228 | } 229 | qr.Rows = append(qr.Rows, vals) 230 | } 231 | err = rows.Err() 232 | if err != nil { 233 | qr.Rows = nil 234 | qr.Error = err.Error() 235 | } 236 | 237 | return 238 | } 239 | 240 | // doExec runs a sql query with args on a pgxpool connection. It returns the 241 | // rows affected or the error in an execResult. 242 | func doExec(ctx context.Context, conn *pgxpool.Conn, query string, args ...any) (er execResult) { 243 | tag, err := conn.Exec(ctx, query, args...) 244 | if err != nil { 245 | er.Error = err.Error() 246 | return 247 | } 248 | er.RowsAffected = tag.RowsAffected() 249 | return 250 | } 251 | 252 | // parseArgs checks the arguments passed to the acquire.query() and 253 | // acquire.exec() js functions. 254 | func parseArgs(f string, args []qjs.Value) (q string, sqlArgs []any, err error) { 255 | // check arg count 256 | if len(args) < 1 { 257 | err = fmt.Errorf("$sys.%s: need at least 1 argument", f) 258 | return 259 | } 260 | 261 | // first arg: sql query 262 | if args[0].Tag() != qjs.TagString { 263 | err = fmt.Errorf("$sys.%s: first argument must be a SQL query (string)", f) 264 | return 265 | } 266 | q, _ = args[0].Any().(string) 267 | 268 | // rest of the args are passed to sql query 269 | sqlArgs = make([]any, len(args)-1) 270 | for i := 1; i < len(args); i++ { 271 | sqlArgs[i-1] = args[i].Any() 272 | } 273 | return 274 | } 275 | 276 | //------------------------------------------------------------------------------ 277 | 278 | func (a *APIServer) runScriptHandler(resp http.ResponseWriter, req *http.Request, 279 | ep *Endpoint, params []any, logger zerolog.Logger) { 280 | 281 | // convert params to a map 282 | paramsMap := make(map[string]any, len(ep.Params)) 283 | for i := range ep.Params { 284 | paramsMap[ep.Params[i].Name] = params[i] 285 | } 286 | 287 | // actually run the script 288 | result, tag, err := a.runScript(ep.Script, paramsMap, logger, ep.Debug) 289 | 290 | // helper function to write string/object results 291 | writeResult := func(code int) bool { 292 | // is it a string? 293 | if tag == qjs.TagString { 294 | resp.Header().Set("Content-Type", "text/plain; charset=utf8") 295 | resp.WriteHeader(code) 296 | if _, err := resp.Write([]byte(result.(string))); err != nil { 297 | logger.Error().Err(err).Msg("error writing response") 298 | } 299 | return true 300 | } 301 | 302 | // else we'll also accept an object, as long as it is not an array 303 | if tag == qjs.TagObject { 304 | if _, ok := result.([]any); !ok { 305 | resp.Header().Set("Content-Type", "application/json") 306 | resp.WriteHeader(code) 307 | enc := json.NewEncoder(resp) 308 | enc.SetIndent("", " ") 309 | if err := enc.Encode(result); err != nil { 310 | logger.Error().Err(err).Msg("error encoding result object") 311 | } 312 | return true 313 | } 314 | } 315 | 316 | return false 317 | } 318 | 319 | write500Body := func(body string) { 320 | resp.WriteHeader(500) 321 | if _, err := resp.Write([]byte(body)); err != nil { 322 | logger.Error().Err(err).Msg("error writing response") 323 | } 324 | } 325 | 326 | // did we get any result at all? 327 | noResult := tag == qjs.TagUndefined || tag == qjs.TagUninitialized || tag == qjs.TagNull 328 | 329 | // did it fail? 330 | if err != nil { 331 | if noResult { 332 | logger.Error().Err(err).Msg("script failed") 333 | write500Body(err.Error()) 334 | } else { 335 | if writeResult(500) { 336 | logger.Error().Err(err).Msg("script failed with result") 337 | } else { 338 | logger.Error().Msg("script failed, also unsupported result type from script") 339 | write500Body("script error") 340 | } 341 | } 342 | return 343 | } 344 | 345 | // success, but no $sys.result 346 | if noResult { 347 | resp.WriteHeader(204) 348 | return 349 | } 350 | 351 | // success with $sys.result of supported type 352 | if writeResult(200) { 353 | return 354 | } 355 | 356 | // $sys.result is not usable 357 | http.Error(resp, "unsupported result type from script", http.StatusInternalServerError) 358 | logger.Error().Msg("unsupported result type from script") 359 | } 360 | 361 | func (a *APIServer) runScript(script string, paramsMap map[string]any, 362 | logger zerolog.Logger, debug bool) (result any, tag int, err error) { 363 | // make the quickjs code run entirely on the same thread 364 | runtime.LockOSThread() 365 | defer runtime.UnlockOSThread() 366 | 367 | // setup javascript env 368 | rt := qjs.NewRuntime() 369 | ctx := rt.NewContext() 370 | sctx := newScriptContext(ctx, a, logger, debug) 371 | 372 | // create and set the $sys object 373 | sys := ctx.Object() 374 | // set params 375 | paramsObj, _ := ctx.ObjectViaJSON(paramsMap) 376 | sys.SetProperty("params", paramsObj) 377 | // set acquire 378 | setfnProp(ctx, sys, "acquire", sctx.acquire) 379 | // set into global 380 | global := ctx.Global() 381 | global.SetProperty(jsPropSys, sys.Dup()) 382 | 383 | // call rti hook 384 | if a.rti != nil && a.rti.InitJSCtx != nil { 385 | a.rti.InitJSCtx(ctx) 386 | } 387 | 388 | // run the script 389 | obj, errObj := ctx.Eval(script) 390 | obj.Free() // obj is the last evaluated expression of script, discard 391 | 392 | // on success, return $sys.result object as "result" 393 | if errObjTag := errObj.Tag(); errObjTag == qjs.TagUndefined { 394 | resultObj := sys.GetProperty("result") 395 | tag = resultObj.Tag() 396 | result = resultObj.Any() 397 | resultObj.Free() 398 | } else { 399 | // else return the errObj as "result", also set "err" 400 | tag = errObjTag 401 | result = errObj.Any() 402 | errObj.Free() 403 | msg := fmt.Sprintf("%v", result) 404 | if len(msg) == 0 { 405 | msg = "script error" 406 | } 407 | err = errors.New(msg) 408 | } 409 | 410 | // cleanup 411 | sctx.close() 412 | sys.Free() 413 | global.Free() 414 | ctx.Free() 415 | rt.RunGC() 416 | rt.Free() 417 | return 418 | } 419 | -------------------------------------------------------------------------------- /script_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows_test 18 | 19 | import ( 20 | "testing" 21 | "time" 22 | 23 | "github.com/stretchr/testify/require" 24 | ) 25 | 26 | const cfgTestScriptBasic = `{ 27 | "version": "1", 28 | "listen": "127.0.0.1:60000", 29 | "endpoints": [ 30 | { 31 | "uri": "/acquire-1", 32 | "implType": "javascript", 33 | "script": "$sys.acquire(1,2,3)", 34 | "debug": true 35 | }, 36 | { 37 | "uri": "/acquire-2", 38 | "implType": "javascript", 39 | "script": "$sys.acquire(1)", 40 | "debug": true 41 | }, 42 | { 43 | "uri": "/acquire-3", 44 | "implType": "javascript", 45 | "script": "$sys.acquire('')", 46 | "debug": true 47 | }, 48 | { 49 | "uri": "/acquire-4", 50 | "implType": "javascript", 51 | "script": "$sys.acquire('default', 10)", 52 | "debug": true 53 | }, 54 | { 55 | "uri": "/acquire-5", 56 | "implType": "javascript", 57 | "script": "$sys.acquire('default', 10.5)", 58 | "debug": true 59 | }, 60 | { 61 | "uri": "/acquire-6", 62 | "implType": "javascript", 63 | "script": "$sys.acquire('default', 'bad')", 64 | "debug": true 65 | }, 66 | { 67 | "uri": "/acquire-7", 68 | "implType": "javascript", 69 | "script": "$sys.acquire('nosuchdatasource', 10)", 70 | "debug": true 71 | }, 72 | 73 | { 74 | "uri": "/setup", 75 | "implType": "exec", 76 | "script": "drop table if exists movies; create table movies (name text, year integer); insert into movies values ('The Shawshank Redemption', 1994), ('The Godfather', 1972), ('The Dark Knight', 2008), ('The Godfather Part II', 1974), ('12 Angry Men', 1957);", 77 | "datasource": "default" 78 | }, 79 | { 80 | "uri": "/query-1", 81 | "implType": "javascript", 82 | "script": "$sys.acquire('default').query()", 83 | "debug": true 84 | }, 85 | { 86 | "uri": "/query-2", 87 | "implType": "javascript", 88 | "script": "$sys.acquire('default').query(100)", 89 | "debug": true 90 | }, 91 | { 92 | "uri": "/query-3", 93 | "implType": "javascript", 94 | "script": "$sys.acquire('default').query('select * from movies where year=$1', 42)", 95 | "debug": true 96 | }, 97 | 98 | { 99 | "uri": "/exec-1/{year}", 100 | "implType": "javascript", 101 | "script": "$sys.acquire('default').exec('update movies set year=year where year=$1', $sys.params.year)", 102 | "debug": true, 103 | "params": [ 104 | { 105 | "name": "year", 106 | "type": "integer", 107 | "in": "path", 108 | "minimum": 1900, "maximum": 2050, 109 | "required": true 110 | } 111 | ] 112 | }, 113 | { 114 | "uri": "/exec-2", 115 | "implType": "javascript", 116 | "script": "$sys.acquire('default').exec('** syntax error')", 117 | "debug": true 118 | }, 119 | { 120 | "uri": "/exec-3", 121 | "implType": "javascript", 122 | "script": "$sys.acquire('default').exec(42)", 123 | "debug": true 124 | }, 125 | 126 | { 127 | "uri": "/result-1", 128 | "implType": "javascript", 129 | "script": "$sys.result = 'foo'", 130 | "debug": true 131 | }, 132 | { 133 | "uri": "/result-2", 134 | "implType": "javascript", 135 | "script": "$sys.result = ['foo']", 136 | "debug": true 137 | } 138 | ], 139 | "datasources": [ { "name": "default" } ] 140 | }` 141 | 142 | func TestScriptAcquire(t *testing.T) { 143 | r := require.New(t) 144 | 145 | cfg := loadCfg(r, cfgTestScriptBasic) 146 | s := startServerFull(r, cfg) 147 | 148 | body, resp := doGet(r, "http://127.0.0.1:60000/acquire-1") 149 | r.NotNil(resp) 150 | r.Equal(500, resp.StatusCode) 151 | r.Equal("{\n \"Message\": \"Error: $sys.acquire: needs 1 or 2 arguments\"\n}\n", string(body)) 152 | 153 | body, resp = doGet(r, "http://127.0.0.1:60000/acquire-2") 154 | r.NotNil(resp) 155 | r.Equal(500, resp.StatusCode) 156 | r.Equal("{\n \"Message\": \"Error: $sys.acquire: first argument must be datasource name (string)\"\n}\n", string(body)) 157 | 158 | body, resp = doGet(r, "http://127.0.0.1:60000/acquire-3") 159 | r.NotNil(resp) 160 | r.Equal(500, resp.StatusCode) 161 | r.Equal("{\n \"Message\": \"Error: $sys.acquire: datasource not specified\"\n}\n", string(body)) 162 | 163 | body, resp = doGet(r, "http://127.0.0.1:60000/acquire-4") 164 | r.NotNil(resp) 165 | r.Equal(204, resp.StatusCode) 166 | r.Equal(0, len(body)) 167 | 168 | body, resp = doGet(r, "http://127.0.0.1:60000/acquire-5") 169 | r.NotNil(resp) 170 | r.Equal(204, resp.StatusCode) 171 | r.Equal(0, len(body)) 172 | 173 | body, resp = doGet(r, "http://127.0.0.1:60000/acquire-6") 174 | r.NotNil(resp) 175 | r.Equal(500, resp.StatusCode) 176 | r.Equal("{\n \"Message\": \"Error: $sys.acquire: second argument must be timeout in seconds (number)\"\n}\n", string(body)) 177 | 178 | body, resp = doGet(r, "http://127.0.0.1:60000/acquire-7") 179 | r.NotNil(resp) 180 | r.Equal(500, resp.StatusCode) 181 | r.Equal("{\n \"Message\": \"Error: $sys.acquire: unknown datasource \\\"nosuchdatasource\\\"\"\n}\n", string(body)) 182 | 183 | checkGetOK(r, "http://127.0.0.1:60000/setup") 184 | 185 | body, resp = doGet(r, "http://127.0.0.1:60000/query-1") 186 | r.NotNil(resp) 187 | r.Equal(500, resp.StatusCode) 188 | r.Equal("{\n \"Message\": \"Error: $sys.query: need at least 1 argument\"\n}\n", string(body)) 189 | 190 | body, resp = doGet(r, "http://127.0.0.1:60000/query-2") 191 | r.NotNil(resp) 192 | r.Equal(500, resp.StatusCode) 193 | r.Equal("{\n \"Message\": \"Error: $sys.query: first argument must be a SQL query (string)\"\n}\n", string(body)) 194 | 195 | body, resp = doGet(r, "http://127.0.0.1:60000/query-3") 196 | r.NotNil(resp) 197 | r.Equal(204, resp.StatusCode) 198 | r.Equal(0, len(body)) 199 | 200 | body, resp = doGet(r, "http://127.0.0.1:60000/exec-1/1972") 201 | r.NotNil(resp) 202 | r.Equal(204, resp.StatusCode) 203 | r.Equal(0, len(body)) 204 | 205 | body, resp = doGet(r, "http://127.0.0.1:60000/exec-2") 206 | r.NotNil(resp) 207 | r.Equal(500, resp.StatusCode) 208 | r.Equal("{\n \"Message\": \"Error: ERROR: syntax error at or near \\\"**\\\" (SQLSTATE 42601)\"\n}\n", string(body)) 209 | 210 | body, resp = doGet(r, "http://127.0.0.1:60000/exec-3") 211 | r.NotNil(resp) 212 | r.Equal(500, resp.StatusCode) 213 | r.Equal("{\n \"Message\": \"Error: $sys.exec: first argument must be a SQL query (string)\"\n}\n", string(body)) 214 | 215 | body, resp = doGet(r, "http://127.0.0.1:60000/result-1") 216 | r.NotNil(resp) 217 | r.Equal(200, resp.StatusCode) 218 | r.Equal("foo", string(body)) 219 | 220 | body, resp = doGet(r, "http://127.0.0.1:60000/result-2") 221 | r.NotNil(resp) 222 | r.Equal(500, resp.StatusCode) 223 | r.Equal("unsupported result type from script\n", string(body)) 224 | 225 | s.Stop(time.Second) 226 | } 227 | -------------------------------------------------------------------------------- /server_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows_test 18 | 19 | import ( 20 | "io" 21 | "net" 22 | "net/url" 23 | "sync" 24 | "testing" 25 | "time" 26 | 27 | "github.com/rapidloop/rapidrows" 28 | "github.com/rapidloop/rapidrows/qjs" 29 | "github.com/rs/zerolog" 30 | "github.com/stretchr/testify/require" 31 | ) 32 | 33 | func TestServerInvalidCfg(t *testing.T) { 34 | r := require.New(t) 35 | 36 | s, err := rapidrows.NewAPIServer(nil, nil) 37 | r.Nil(s) 38 | r.NotNil(err) 39 | 40 | cfg := rapidrows.APIServerConfig{} 41 | s, err = rapidrows.NewAPIServer(&cfg, nil) 42 | r.Nil(s) 43 | r.NotNil(err) 44 | } 45 | 46 | const cfgTestServerBasic = `{ 47 | "version": "1", 48 | "listen": "127.0.0.1:60000", 49 | "compression": true, 50 | "cors": { "debug": true, "maxAge": 3600 }, 51 | "endpoints": [ 52 | { 53 | "uri": "/movies", 54 | "methods": [ "GET" ], 55 | "implType": "query-json", 56 | "script": "select * from movies order by year desc", 57 | "datasource": "default", 58 | "debug": true, 59 | "timeout": 60, 60 | "cache": 1 61 | }, 62 | { 63 | "uri": "/movies-in-year/{year}", 64 | "implType": "query-json", 65 | "script": "select * from movies where year = $1 order by year desc", 66 | "datasource": "default", 67 | "cache": 1, 68 | "debug": true, 69 | "params": [ 70 | { 71 | "name": "year", 72 | "in": "path", 73 | "type": "integer", 74 | "minimum": 1950, 75 | "maximum": 2030 76 | } 77 | ] 78 | }, 79 | { 80 | "uri": "/movies-in-year-no-cache/{year}", 81 | "implType": "query-json", 82 | "script": "select * from movies where year = $1 order by year desc", 83 | "datasource": "default", 84 | "params": [ 85 | { 86 | "name": "year", 87 | "in": "path", 88 | "type": "integer", 89 | "minimum": 1950, 90 | "maximum": 2030 91 | } 92 | ] 93 | }, 94 | { 95 | "uri": "/movies-csv", 96 | "implType": "query-csv", 97 | "script": "select * from movies order by year desc", 98 | "datasource": "default", 99 | "cache": 3600 100 | }, 101 | { 102 | "uri": "/setup", 103 | "implType": "exec", 104 | "script": "drop table if exists movies; create table movies (name text, year integer); insert into movies values ('The Shawshank Redemption', 1994), ('The Godfather', 1972), ('The Dark Knight', 2008), ('The Godfather Part II', 1974), ('12 Angry Men', 1957);", 105 | "datasource": "default", 106 | "debug": true, 107 | "timeout": 5 108 | }, 109 | { 110 | "uri": "/movies-js", 111 | "implType": "javascript", 112 | "script": "$sys.result=$sys.acquire('default').query('select * from movies order by year desc')" 113 | }, 114 | { 115 | "uri": "/movies-tx1", 116 | "implType": "exec", 117 | "tx": { "access": "read write", "level": "serializable" }, 118 | "script": "update movies set name='x' where name='y'", 119 | "datasource": "default" 120 | }, 121 | { 122 | "uri": "/info-json", 123 | "implType": "static-json", 124 | "script": "{\"apiVersion\": 1}" 125 | }, 126 | { 127 | "uri": "/exec-error", 128 | "implType": "exec", 129 | "script": "syntax error", 130 | "datasource": "default" 131 | }, 132 | { 133 | "uri": "/query-error", 134 | "implType": "query-json", 135 | "script": "syntax error", 136 | "datasource": "default" 137 | }, 138 | { 139 | "uri": "/script-error-1", 140 | "implType": "javascript", 141 | "script": "*** syntax error" 142 | }, 143 | { 144 | "uri": "/script-error-2", 145 | "implType": "javascript", 146 | "script": "$sys.acquire('no.such')" 147 | }, 148 | { 149 | "uri": "/script-error-3", 150 | "implType": "javascript", 151 | "script": "$sys.acquire('default').query('syntax error')" 152 | }, 153 | { 154 | "uri": "/script-error-4", 155 | "implType": "javascript", 156 | "script": "throw 'foo'" 157 | }, 158 | { 159 | "uri": "/test-cache", 160 | "implType": "query-json", 161 | "script": "select $1 || array_to_string($2::text[], '')", 162 | "datasource": "default", 163 | "cache": 60, 164 | "params": [ 165 | { 166 | "name": "param1", 167 | "in": "body", 168 | "type": "string" 169 | }, 170 | { 171 | "name": "param2", 172 | "in": "body", 173 | "type": "array", 174 | "elemType": "string" 175 | } 176 | ] 177 | }, 178 | { 179 | "uri": "/test-csv-no-rows", 180 | "implType": "query-csv", 181 | "script": "select * from movies where year=1900", 182 | "datasource": "default" 183 | } 184 | ], 185 | "streams": [ 186 | { 187 | "uri": "/movies-changes", 188 | "type": "sse", 189 | "channel": "movieschanges", 190 | "datasource": "default", 191 | "debug": true 192 | }, 193 | { 194 | "uri": "/movies-changes2", 195 | "type": "sse", 196 | "channel": "movieschanges", 197 | "datasource": "default" 198 | }, 199 | { 200 | "uri": "/movies-changes3", 201 | "type": "sse", 202 | "channel": "movieschanges3", 203 | "datasource": "default" 204 | } 205 | ], 206 | "datasources": [ 207 | { 208 | "name": "default", 209 | "timeout": 5 210 | } 211 | ] 212 | }` 213 | 214 | const expMoviesJson = `{ 215 | "rows": [ 216 | [ 217 | "The Dark Knight", 218 | 2008 219 | ], 220 | [ 221 | "The Shawshank Redemption", 222 | 1994 223 | ], 224 | [ 225 | "The Godfather Part II", 226 | 1974 227 | ], 228 | [ 229 | "The Godfather", 230 | 1972 231 | ], 232 | [ 233 | "12 Angry Men", 234 | 1957 235 | ] 236 | ] 237 | } 238 | ` 239 | 240 | const expMoviesCsv = `The Dark Knight,2008 241 | The Shawshank Redemption,1994 242 | The Godfather Part II,1974 243 | The Godfather,1972 244 | 12 Angry Men,1957 245 | ` 246 | 247 | const expMoviesTx1 = `{ 248 | "rowsAffected": 0 249 | } 250 | ` 251 | 252 | const expMoviesInYear = `{ 253 | "rows": [ 254 | [ 255 | "The Godfather", 256 | 1972 257 | ] 258 | ] 259 | } 260 | ` 261 | 262 | const expMoviesTestCache = `{ 263 | "rows": [ 264 | [ 265 | "foobarbaz" 266 | ] 267 | ] 268 | } 269 | ` 270 | 271 | func checkGetOK(r *require.Assertions, u string) { 272 | _, resp := doGet(r, u) 273 | r.Equal(200, resp.StatusCode) 274 | } 275 | 276 | func startServerFull(r *require.Assertions, cfg *rapidrows.APIServerConfig, dest ...io.Writer) *rapidrows.APIServer { 277 | var cache sync.Map 278 | cacheSet := func(key uint64, value []byte) { 279 | if len(value) == 0 { 280 | cache.Delete(key) 281 | } else { 282 | cache.Store(key, value) 283 | } 284 | } 285 | cacheGet := func(key uint64) (value []byte, found bool) { 286 | if v, ok := cache.Load(key); ok && v != nil { 287 | return v.([]byte), true 288 | } 289 | return nil, false 290 | } 291 | var logger zerolog.Logger 292 | if len(dest) > 0 { 293 | logger = zerolog.New(dest[0]) 294 | } else { 295 | logger = zerolog.Nop() 296 | } 297 | rti := &rapidrows.RuntimeInterface{ 298 | Logger: &logger, 299 | CacheSet: cacheSet, 300 | CacheGet: cacheGet, 301 | ReportMetric: func(name string, labels []string, value float64) {}, 302 | InitJSCtx: func(ctx *qjs.Context) {}, 303 | } 304 | s, err := rapidrows.NewAPIServer(cfg, rti) 305 | r.NotNil(s, "error was %v", err) 306 | r.Nil(err) 307 | r.Nil(s.Start()) 308 | return s 309 | } 310 | 311 | func TestServerBasic(t *testing.T) { 312 | r := require.New(t) 313 | 314 | cfg := loadCfg(r, cfgTestServerBasic) 315 | s := startServerFull(r, cfg) 316 | 317 | checkGetOK(r, "http://127.0.0.1:60000/setup") 318 | 319 | body, resp := doGet(r, "http://127.0.0.1:60000/movies") 320 | r.Equal(expMoviesJson, string(body)) 321 | r.Equal(200, resp.StatusCode) 322 | 323 | // repeat both again, for testing cache 324 | body, resp = doGet(r, "http://127.0.0.1:60000/movies") 325 | r.Equal(expMoviesJson, string(body)) 326 | r.Equal(200, resp.StatusCode) 327 | 328 | // wait for cache expiry 329 | time.Sleep(time.Second) 330 | body, resp = doGet(r, "http://127.0.0.1:60000/movies") 331 | r.Equal(expMoviesJson, string(body)) 332 | r.Equal(200, resp.StatusCode) 333 | 334 | body, resp = doGet(r, "http://127.0.0.1:60000/movies-csv") 335 | r.Equal(expMoviesCsv, string(body)) 336 | r.Equal(200, resp.StatusCode) 337 | 338 | body, resp = doGet(r, "http://127.0.0.1:60000/movies-csv") 339 | r.Equal(expMoviesCsv, string(body)) 340 | r.Equal(200, resp.StatusCode) 341 | 342 | body, resp = doGet(r, "http://127.0.0.1:60000/movies-js") 343 | r.Equal(expMoviesJson, string(body)) 344 | r.Equal(200, resp.StatusCode) 345 | 346 | body, resp = doGet(r, "http://127.0.0.1:60000/movies-tx1") 347 | r.Equal(expMoviesTx1, string(body)) 348 | r.Equal(200, resp.StatusCode) 349 | 350 | body, resp = doGet(r, "http://127.0.0.1:60000/movies-in-year/1972") 351 | r.Equal(expMoviesInYear, string(body)) 352 | r.Equal(200, resp.StatusCode) 353 | 354 | // repeat for cache 355 | body, resp = doGet(r, "http://127.0.0.1:60000/movies-in-year/1972") 356 | r.Equal(expMoviesInYear, string(body)) 357 | r.Equal(200, resp.StatusCode) 358 | 359 | body, resp = doGet(r, "http://127.0.0.1:60000/movies-in-year-no-cache/1972") 360 | r.Equal(expMoviesInYear, string(body)) 361 | r.Equal(200, resp.StatusCode) 362 | 363 | body, resp = doGet(r, "http://127.0.0.1:60000/info-json") 364 | r.Equal(`{"apiVersion": 1}`, string(body)) 365 | r.Equal(200, resp.StatusCode) 366 | r.Equal("application/json", resp.Header.Get("Content-Type")) 367 | 368 | // query-json with a string param and cache 369 | // query-json with a []string param and cache 370 | v := url.Values{} 371 | v.Add("param1", "foo") 372 | v.Add("param2", "bar") 373 | v.Add("param2", "baz") 374 | body, resp = doPostForm(r, "http://127.0.0.1:60000/test-cache", v) 375 | r.Equal(expMoviesTestCache, string(body)) 376 | r.Equal(200, resp.StatusCode) 377 | body, resp = doPostForm(r, "http://127.0.0.1:60000/test-cache", v) 378 | r.Equal(expMoviesTestCache, string(body)) 379 | r.Equal(200, resp.StatusCode) 380 | 381 | // query-csv that returns no rows 382 | body, resp = doGet(r, "http://127.0.0.1:60000/test-csv-no-rows") 383 | r.Equal("", string(body)) 384 | r.Equal(200, resp.StatusCode) 385 | 386 | s.Stop(time.Second) 387 | } 388 | 389 | const expExecError = `{"rowsAffected":0,"error":"ERROR: syntax error at or near \"syntax\" (SQLSTATE 42601)"} 390 | ` 391 | 392 | const expQueryError = `{ 393 | "rows": null, 394 | "error": "ERROR: syntax error at or near \"syntax\" (SQLSTATE 42601)" 395 | } 396 | ` 397 | 398 | func TestServerErrors(t *testing.T) { 399 | r := require.New(t) 400 | 401 | cfg := loadCfg(r, cfgTestServerBasic) 402 | s := startServerFull(r, cfg) 403 | 404 | checkGetOK(r, "http://127.0.0.1:60000/setup") 405 | 406 | body, resp := doGet(r, "http://127.0.0.1:60000/exec-error") 407 | r.Equal(expExecError, string(body)) 408 | r.Equal(500, resp.StatusCode) 409 | r.Equal("application/json", resp.Header.Get("Content-Type")) 410 | 411 | body, resp = doGet(r, "http://127.0.0.1:60000/query-error") 412 | r.Equal(expQueryError, string(body)) 413 | r.Equal(500, resp.StatusCode) 414 | r.Equal("application/json", resp.Header.Get("Content-Type")) 415 | 416 | _, resp = doGet(r, "http://127.0.0.1:60000/script-error-1") 417 | r.Equal(500, resp.StatusCode) 418 | 419 | _, resp = doGet(r, "http://127.0.0.1:60000/script-error-2") 420 | r.Equal(500, resp.StatusCode) 421 | 422 | _, resp = doGet(r, "http://127.0.0.1:60000/script-error-3") 423 | r.Equal(500, resp.StatusCode) 424 | 425 | _, resp = doGet(r, "http://127.0.0.1:60000/script-error-4") 426 | r.Equal(500, resp.StatusCode) 427 | 428 | s.Stop(time.Second) 429 | } 430 | 431 | const cfgTestServerBadDS = `{ 432 | "version": "1", 433 | "listen": "127.0.0.1:60000", 434 | "datasources": [ { "name": "foo", "host": "192.0.0.1", "timeout": 2}] 435 | }` 436 | 437 | func TestServerStartupErrors(t *testing.T) { 438 | r := require.New(t) 439 | 440 | cfg := loadCfg(r, cfgTestServerBadDS) 441 | logger := zerolog.Nop() 442 | rti := &rapidrows.RuntimeInterface{ 443 | Logger: &logger, 444 | } 445 | s, err := rapidrows.NewAPIServer(cfg, rti) 446 | r.NotNil(s, "error was %v", err) 447 | r.Nil(err) 448 | r.NotNil(s.Start()) 449 | } 450 | 451 | const cfgTestServerNoPort = `{ 452 | "version": "1", 453 | "listen": "127.0.0.1" 454 | }` 455 | 456 | func TestServerStartupMisc(t *testing.T) { 457 | r := require.New(t) 458 | 459 | // (try to) start a listener on 8080, the start APIServer on default 460 | // port (which should be 8080) 461 | lnr, _ := net.Listen("tcp", "127.0.0.1:8080") 462 | cfg := loadCfg(r, cfgTestServerNoPort) 463 | s, err := rapidrows.NewAPIServer(cfg, nil) 464 | r.NotNil(s) 465 | r.Nil(err) 466 | err = s.Start() 467 | r.EqualError(err, "listen tcp 127.0.0.1:8080: bind: address already in use") 468 | if lnr != nil { 469 | lnr.Close() 470 | } 471 | 472 | // stop a server that is not started successfully 473 | r.Nil(s.Stop(time.Second)) 474 | } 475 | -------------------------------------------------------------------------------- /streams.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows 18 | 19 | import ( 20 | "context" 21 | "errors" 22 | "fmt" 23 | "io" 24 | "net/http" 25 | "strings" 26 | "sync" 27 | "sync/atomic" 28 | "time" 29 | 30 | "github.com/go-chi/chi/v5" 31 | "github.com/jackc/pgconn" 32 | "github.com/jackc/pgx/v4" 33 | "github.com/rs/zerolog" 34 | "nhooyr.io/websocket" 35 | ) 36 | 37 | func (a *APIServer) startNotifDispatchers() error { 38 | // make a list of all channels that each datasource needs 39 | ds2pgchans := make(map[string][]string) 40 | add := func(ds, pgchan string) { 41 | if pgchans, ok := ds2pgchans[ds]; ok { 42 | for _, c := range pgchans { 43 | if c == pgchan { 44 | return // already present 45 | } 46 | } 47 | ds2pgchans[ds] = append(pgchans, pgchan) 48 | } else { 49 | ds2pgchans[ds] = []string{pgchan} 50 | } 51 | } 52 | for _, s := range a.cfg.Streams { 53 | add(s.Datasource, s.Channel) 54 | } 55 | 56 | // open a long-lived connection to each datasource, and start a notifDispatcher 57 | // on that 58 | connsToClose := make([]*pgx.Conn, 0, len(ds2pgchans)) 59 | var err error 60 | for ds, pgchans := range ds2pgchans { 61 | nd := newNotifDispatcher(pgchans, a.logger) 62 | var conn *pgx.Conn 63 | conn, err = a.ds.hijack(ds) 64 | if err != nil { 65 | a.logger.Error().Str("datasource", ds).Err(err). 66 | Msg("failed to open connection") 67 | break 68 | } 69 | if err = nd.start(conn); err != nil { // note: err =, not err := 70 | a.logger.Error().Str("datasource", ds).Err(err). 71 | Msg("failed to start notification dispatcher") 72 | break 73 | } 74 | a.nd.Store(ds, nd) 75 | connsToClose = append(connsToClose, conn) 76 | a.logger.Info().Str("datasource", ds).Strs("channels", pgchans). 77 | Msg("started notification dispatcher") 78 | } 79 | 80 | // on errors, close all opened connections 81 | if err != nil { 82 | for _, c := range connsToClose { 83 | c.Close(context.Background()) 84 | } 85 | return err 86 | } 87 | 88 | return nil 89 | } 90 | 91 | func (a *APIServer) stopNotifDispatchers() { 92 | a.nd.Range(func(k, v any) bool { 93 | v.(*notifDispatcher).stop() 94 | a.logger.Info().Str("datasource", k.(string)).Msg("stopped notification dispatcher") 95 | return true 96 | }) 97 | } 98 | 99 | func (a *APIServer) setupStream(r *chi.Mux, s *Stream) { 100 | var handler http.HandlerFunc = func(resp http.ResponseWriter, req *http.Request) { 101 | a.serveStream(resp, req, s) 102 | } 103 | 104 | r.HandleFunc(a.cfg.CommonPrefix+s.URI, handler) 105 | } 106 | 107 | func (a *APIServer) serveStream(resp http.ResponseWriter, req *http.Request, s *Stream) { 108 | // setup logger, debug logging 109 | logger := a.logger.With().Str("endpoint", a.cfg.CommonPrefix+s.URI).Logger() 110 | if s.Debug { 111 | logger.Debug().Str("channel", s.Channel).Str("datasource", s.Datasource). 112 | Str("type", s.Type).Msg("stream handler start") 113 | } 114 | 115 | // discard body, ignore errors 116 | _, _ = io.CopyN(io.Discard, req.Body, 4096) 117 | 118 | // get notifdispatcher for the stream's datasource 119 | var nd *notifDispatcher 120 | if v, ok := a.nd.Load(s.Datasource); ok && v != nil { 121 | nd = v.(*notifDispatcher) 122 | } 123 | if nd == nil { 124 | // should not happen 125 | logger.Error().Str("datasource", s.Datasource). 126 | Msg("internal error: notification dispatcher not found") 127 | http.Error(resp, "internal error", http.StatusInternalServerError) 128 | return 129 | } 130 | 131 | // do the main loop 132 | nw := newNotifWriter() 133 | nd.register(s.Channel, nw) 134 | var err error 135 | if s.Type == "websocket" { 136 | err = nw.loopWS(a.bgctx, resp, req, nil, false, a.logger) 137 | } else { 138 | err = nw.loopSSE(a.bgctx, resp, req, a.logger) 139 | } 140 | if !errors.Is(err, context.Canceled) { 141 | nd.unregister(s.Channel, nw) 142 | } 143 | // if bgctx was cancelled, that means the server is shutting down and 144 | // notifDispatcher might have gone away already 145 | 146 | // don't consider 'broken pipe' and 'i/o timeout' as errors to be logged 147 | if err != nil { 148 | if s := err.Error(); strings.Contains(s, "broken pipe") || 149 | strings.Contains(s, "i/o timeout") { 150 | err = nil 151 | } 152 | } 153 | 154 | if err != nil { 155 | logger.Error().Err(err).Msg("stream closed on error") 156 | } else if s.Debug { 157 | logger.Debug().Str("channel", s.Channel).Str("datasource", s.Datasource). 158 | Str("type", s.Type).Msg("stream handler end") 159 | } 160 | } 161 | 162 | //------------------------------------------------------------------------------ 163 | 164 | // notifWriter writes out the payloads of pgconn.Notification objects into 165 | // a *websocket.Conn. It does not have a dedicated goroutine, it's event loop 166 | // is meant to be hosted by the http handler goroutine. 167 | type notifWriter struct { 168 | q chan string 169 | qClosed bool 170 | qMtx sync.Mutex 171 | } 172 | 173 | // notifWriterBacklog is the max number of notifications that are allowed to 174 | // be pending to write into the websocket. If a new notification is available 175 | // and we still have these many waiting to be written, the websocket is closed. 176 | const notifWriterBacklog = 16 177 | 178 | func newNotifWriter() *notifWriter { 179 | return ¬ifWriter{ 180 | q: make(chan string, notifWriterBacklog), 181 | } 182 | } 183 | 184 | // accept takes in a new notification. This must NOT block. It is called by 185 | // the notifDispatcher. There is a race between client disconnects for various 186 | // reasons and a new notification arriving, so handle the case that when we 187 | // attempt to write to the channel or close it, it is already closed by the 188 | // other goroutine. Alternative is to route a special message via the 189 | // notifDispatcher and close the channel only here; but then there is also 190 | // the case that the notifDispatcher might have gone at server exit, and only 191 | // we are alive. 192 | func (n *notifWriter) accept(payload string) { 193 | defer func() { 194 | if r := recover(); r != nil { 195 | if err, _ := r.(error); err != nil { 196 | if err.Error() == "send on closed channel" { 197 | n.closeQ() 198 | } 199 | } 200 | } 201 | }() 202 | 203 | select { 204 | case n.q <- payload: 205 | default: 206 | // our queue is full, we can't make the caller wait, so we abort 207 | n.closeQ() 208 | } 209 | } 210 | 211 | func (n *notifWriter) closeQ() { 212 | n.qMtx.Lock() 213 | if !n.qClosed { 214 | close(n.q) 215 | n.qClosed = true 216 | } 217 | n.qMtx.Unlock() 218 | } 219 | 220 | var ( 221 | notifWriteTimeout = 10 * time.Second 222 | errTooSlow = errors.New("aborting connection because it is too slow") 223 | ) 224 | 225 | // loopWS upgrades the given connection to a websocket and writes out the pg 226 | // notifications into it. This is meant to be called directly from the http 227 | // handler goroutine. It will block until client disconnects or if there are 228 | // other errors. notifWriter must not be reused after this exits. 229 | func (n *notifWriter) loopWS(ctx context.Context, resp http.ResponseWriter, 230 | req *http.Request, origins []string, compression bool, 231 | logger zerolog.Logger) error { 232 | 233 | // close q if required when we exit 234 | qclosed := false 235 | defer func() { 236 | if !qclosed { 237 | n.closeQ() 238 | } 239 | }() 240 | 241 | // upgrade connection 242 | ws, err := websocket.Accept(resp, req, &websocket.AcceptOptions{ 243 | InsecureSkipVerify: len(origins) == 0, 244 | OriginPatterns: origins, 245 | CompressionMode: pick(compression, websocket.CompressionContextTakeover, websocket.CompressionDisabled), 246 | }) 247 | if err != nil { 248 | return err 249 | } 250 | defer ws.Close(websocket.StatusInternalError, "") // no-op if already closed 251 | 252 | // start a reader that will respond to pings, but cancels context if any 253 | // other messages are received (we don't expect any) 254 | ctx = ws.CloseRead(ctx) 255 | 256 | for { 257 | select { 258 | 259 | case payload, ok := <-n.q: 260 | if !ok { 261 | ws.Close(websocket.StatusPolicyViolation, "connection too slow") 262 | qclosed = true 263 | return errTooSlow 264 | } 265 | ctx2, cancel := context.WithTimeout(ctx, notifWriteTimeout) 266 | err := ws.Write(ctx2, websocket.MessageText, []byte(payload)) 267 | cancel() 268 | if err != nil { 269 | if cs := websocket.CloseStatus(err); cs == websocket.StatusNormalClosure || cs == websocket.StatusGoingAway { 270 | err = nil 271 | } 272 | return err 273 | } 274 | 275 | case <-ctx.Done(): 276 | ws.Close(websocket.StatusGoingAway, "server shutdown") 277 | return ctx.Err() 278 | } 279 | } 280 | } 281 | 282 | var ( 283 | notifSSEKeepAliveInterval = time.Minute 284 | notifSSEKeepAliveComment = []byte{':', '\n', '\n'} 285 | ) 286 | 287 | // loopSSE is like loopWS, but for server-sent-events. 288 | func (n *notifWriter) loopSSE(ctx context.Context, resp http.ResponseWriter, 289 | req *http.Request, logger zerolog.Logger) error { 290 | // send also a comment every minute to keep the connection alive 291 | ticker := time.NewTicker(notifSSEKeepAliveInterval) 292 | 293 | // cleanup on exit 294 | qclosed := false 295 | defer func() { 296 | if !qclosed { 297 | n.closeQ() 298 | } 299 | ticker.Stop() 300 | }() 301 | 302 | // try to flush data out after each event 303 | flusher, _ := resp.(http.Flusher) 304 | flush := func() { 305 | if flusher != nil { 306 | flusher.Flush() 307 | } 308 | } 309 | 310 | // keep-alive helper 311 | keepalive := func() error { 312 | if _, err := resp.Write(notifSSEKeepAliveComment); err != nil { 313 | return err 314 | } 315 | flush() 316 | return nil 317 | } 318 | 319 | // write out the sse header 320 | h := resp.Header() 321 | h.Set("Content-Type", "text/event-stream") 322 | h.Set("Cache-Control", "no-cache") 323 | h.Set("Connection", "keep-alive") 324 | 325 | // write out an initial comment to start the body 326 | if err := keepalive(); err != nil { 327 | return err 328 | } 329 | 330 | for { 331 | select { 332 | 333 | case <-ticker.C: 334 | if err := keepalive(); err != nil { 335 | return err 336 | } 337 | 338 | case payload, ok := <-n.q: 339 | if !ok { 340 | qclosed = true 341 | return errTooSlow 342 | } 343 | for _, line := range strings.Split(payload, "\n") { 344 | if _, err := fmt.Fprintf(resp, "data: %s\n", line); err != nil { 345 | return err 346 | } 347 | } 348 | if _, err := fmt.Fprintln(resp); err != nil { 349 | return err 350 | } 351 | flush() 352 | 353 | case <-ctx.Done(): 354 | return ctx.Err() 355 | } 356 | } 357 | } 358 | 359 | //------------------------------------------------------------------------------ 360 | 361 | // notifDispatcher listens for notifications from a *pgconn 362 | type notifDispatcher struct { 363 | in chan pgconn.Notification 364 | cmd chan notifDisptacherCmd 365 | pgchans []string 366 | logger zerolog.Logger 367 | wg sync.WaitGroup 368 | stopping atomic.Bool 369 | conn *pgx.Conn 370 | } 371 | 372 | func newNotifDispatcher(pgchans []string, logger zerolog.Logger) *notifDispatcher { 373 | return ¬ifDispatcher{ 374 | in: make(chan pgconn.Notification, 64), 375 | cmd: make(chan notifDisptacherCmd, 1), 376 | pgchans: append([]string{}, pgchans...), 377 | logger: logger, 378 | } 379 | } 380 | 381 | // start starts the notifDispatcher. On success, assumes control of conn; else 382 | // caller has to cleanup conn. 383 | func (nd *notifDispatcher) start(conn *pgx.Conn) error { 384 | // listen to all channels of interest 385 | for _, pgchan := range nd.pgchans { 386 | if _, err := conn.Exec(context.Background(), "LISTEN "+pgchan); err != nil { 387 | return fmt.Errorf("failed to LISTEN to %q: %v", pgchan, err) 388 | } 389 | } 390 | 391 | // ok, conn is ready, store it as a way to stop fetcher 392 | nd.conn = conn 393 | 394 | // start the dispatcher 395 | nd.wg.Add(1) 396 | go nd.dispatcher() 397 | 398 | // start the fetcher 399 | go nd.fetcher(conn) 400 | 401 | return nil 402 | } 403 | 404 | func (nd *notifDispatcher) stop() { 405 | // stop fetcher 406 | nd.stopping.Store(true) 407 | if err := nd.conn.Close(context.Background()); err != nil { 408 | nd.logger.Warn().Err(err).Msg("notification dispatcher: failed to close datasource connection") 409 | } 410 | 411 | // stop dispatcher 412 | nd.cmd <- notifDisptacherCmd{act: actStop} 413 | nd.wg.Wait() 414 | 415 | // cleanup 416 | close(nd.cmd) 417 | close(nd.in) 418 | } 419 | 420 | // fetcher keeps listening to notifications from conn, and pushes them into 421 | // nd.in. To stop the fetcher, close the connection from another goroutine. 422 | func (nd *notifDispatcher) fetcher(conn *pgx.Conn) { 423 | for { 424 | // wait forever for a notification 425 | n, err := conn.WaitForNotification(context.Background()) 426 | if err != nil { 427 | if !nd.stopping.Load() { // ignore error if we're stopping 428 | nd.logger.Error().Err(err).Msg("failed to wait for notification from postgres") 429 | } 430 | return 431 | } 432 | 433 | // hand it off to the notifDispatcher's goroutine 434 | nd.in <- *n 435 | } 436 | } 437 | 438 | const ( 439 | _ = iota 440 | actRegister 441 | actUnregister 442 | actStop 443 | ) 444 | 445 | type notifDisptacherCmd struct { 446 | act int 447 | channel string 448 | writer *notifWriter 449 | } 450 | 451 | func (nd *notifDispatcher) register(pgchan string, writer *notifWriter) { 452 | nd.cmd <- notifDisptacherCmd{act: actRegister, channel: pgchan, writer: writer} 453 | } 454 | 455 | func (nd *notifDispatcher) unregister(pgchan string, writer *notifWriter) { 456 | nd.cmd <- notifDisptacherCmd{act: actUnregister, channel: pgchan, writer: writer} 457 | } 458 | 459 | func (nd *notifDispatcher) dispatcher() { 460 | // a map of pgchan -> all *notifWriter interested in that pgchan 461 | c2ws := make(map[string][]*notifWriter) 462 | unregister := func(c string, w2 *notifWriter) { 463 | if ws, ok := c2ws[c]; ok { 464 | for i, w := range ws { 465 | if w == w2 { 466 | ws[i] = nil 467 | copy(ws[i:], ws[i+1:]) 468 | c2ws[c] = ws[:len(ws)-1] 469 | return 470 | } 471 | } 472 | } 473 | } 474 | 475 | for { 476 | select { 477 | case c := <-nd.cmd: 478 | switch c.act { 479 | case actRegister: 480 | c2ws[c.channel] = append(c2ws[c.channel], c.writer) 481 | case actUnregister: 482 | unregister(c.channel, c.writer) 483 | case actStop: 484 | nd.wg.Done() 485 | return 486 | } 487 | case notif := <-nd.in: 488 | for _, w := range c2ws[notif.Channel] { 489 | w.accept(notif.Payload) 490 | } 491 | } 492 | } 493 | } 494 | 495 | //------------------------------------------------------------------------------ 496 | 497 | func pick[T any](cond bool, ifyes, ifno T) T { 498 | if cond { 499 | return ifyes 500 | } 501 | return ifno 502 | } 503 | -------------------------------------------------------------------------------- /streams_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows_test 18 | 19 | import ( 20 | "bufio" 21 | "context" 22 | "net/http" 23 | "testing" 24 | "time" 25 | 26 | "github.com/stretchr/testify/require" 27 | "nhooyr.io/websocket" 28 | ) 29 | 30 | const cfgTestStreamsBasic = `{ 31 | "version": "1", 32 | "listen": "127.0.0.1:60000", 33 | "endpoints": [ 34 | { 35 | "uri": "/notify/{pgchan}/{payload}", 36 | "implType": "exec", 37 | "script": "select pg_notify($1, $2)", 38 | "datasource": "default", 39 | "params": [ 40 | { 41 | "name": "pgchan", 42 | "in": "path", 43 | "type": "string" 44 | }, 45 | { 46 | "name": "payload", 47 | "in": "path", 48 | "type": "string" 49 | } 50 | ] 51 | } 52 | ], 53 | "streams": [ 54 | { 55 | "uri": "/movies-changes", 56 | "type": "sse", 57 | "channel": "movieschanges", 58 | "datasource": "default", 59 | "debug": true 60 | }, 61 | { 62 | "uri": "/movies-changes2", 63 | "type": "sse", 64 | "channel": "movieschanges", 65 | "datasource": "default" 66 | }, 67 | { 68 | "uri": "/movies-changes3", 69 | "type": "sse", 70 | "channel": "movieschanges3", 71 | "datasource": "default" 72 | }, 73 | { 74 | "uri": "/sse", 75 | "type": "sse", 76 | "channel": "chansse", 77 | "datasource": "default", 78 | "debug": true 79 | }, 80 | { 81 | "uri": "/ws", 82 | "type": "websocket", 83 | "channel": "chanws", 84 | "datasource": "default" 85 | } 86 | ], 87 | "datasources": [ { "name": "default" } ] 88 | }` 89 | 90 | const expResultSSE = `: 91 | 92 | data: foo 93 | 94 | data: bar 95 | 96 | ` 97 | 98 | func TestStreamsSSE(t *testing.T) { 99 | r := require.New(t) 100 | 101 | cfg := loadCfg(r, cfgTestStreamsBasic) 102 | s := startServerFull(r, cfg) 103 | 104 | resp, err := http.Get("http://127.0.0.1:60000/sse") 105 | r.Nil(err) 106 | r.NotNil(resp) 107 | r.Equal(200, resp.StatusCode) 108 | r.NotNil(resp.Body) 109 | 110 | _, resp2 := doGet(r, "http://127.0.0.1:60000/notify/chansse/foo") 111 | r.Equal(200, resp2.StatusCode) 112 | _, resp2 = doGet(r, "http://127.0.0.1:60000/notify/chansse/bar") 113 | r.Equal(200, resp2.StatusCode) 114 | 115 | var result string 116 | scanner := bufio.NewScanner(resp.Body) 117 | n := 1 118 | for n <= 6 && scanner.Scan() { 119 | result += scanner.Text() + "\n" 120 | n++ 121 | } 122 | r.Equal(expResultSSE, result) 123 | 124 | s.Stop(time.Second) 125 | } 126 | 127 | func TestStreamsWS(t *testing.T) { 128 | r := require.New(t) 129 | 130 | cfg := loadCfg(r, cfgTestStreamsBasic) 131 | s := startServerFull(r, cfg) 132 | 133 | conn, resp, err := websocket.Dial(context.Background(), "ws://127.0.0.1:60000/ws", nil) 134 | r.Nil(err) 135 | r.Equal(101, resp.StatusCode) 136 | r.NotNil(conn) 137 | 138 | _, resp2 := doGet(r, "http://127.0.0.1:60000/notify/chanws/foo") 139 | r.Equal(200, resp2.StatusCode) 140 | _, resp2 = doGet(r, "http://127.0.0.1:60000/notify/chanws/bar") 141 | r.Equal(200, resp2.StatusCode) 142 | 143 | mt, data, err := conn.Read(context.Background()) 144 | r.Nil(err) 145 | r.Equal(websocket.MessageText, mt) 146 | r.Equal("foo", string(data)) 147 | 148 | mt, data, err = conn.Read(context.Background()) 149 | r.Nil(err) 150 | r.Equal(websocket.MessageText, mt) 151 | r.Equal("bar", string(data)) 152 | 153 | conn.Close(websocket.StatusInternalError, "bye") 154 | 155 | s.Stop(time.Second) 156 | } 157 | 158 | // TestStreamsWSNoWrite tests whether it is possible to send messages from the 159 | // client to the server (it is not). 160 | func TestStreamsWSNoWrite(t *testing.T) { 161 | r := require.New(t) 162 | 163 | cfg := loadCfg(r, cfgTestStreamsBasic) 164 | s := startServerFull(r, cfg) 165 | 166 | conn, resp, err := websocket.Dial(context.Background(), "ws://127.0.0.1:60000/ws", nil) 167 | r.Nil(err) 168 | r.Equal(101, resp.StatusCode) 169 | r.NotNil(conn) 170 | 171 | err = conn.Write(context.Background(), websocket.MessageText, []byte("baz")) 172 | if err != nil { 173 | r.EqualError(err, `failed to get reader: received close frame: status = StatusPolicyViolation and reason = "unexpected data message"`) 174 | } else { 175 | _, _, err := conn.Read(context.Background()) 176 | r.EqualError(err, `failed to get reader: received close frame: status = StatusPolicyViolation and reason = "unexpected data message"`) 177 | } 178 | 179 | conn.Close(websocket.StatusInternalError, "bye") 180 | 181 | s.Stop(time.Second) 182 | } 183 | 184 | func TestStreamsBadClients(t *testing.T) { 185 | r := require.New(t) 186 | 187 | cfg := loadCfg(r, cfgTestStreamsBasic) 188 | s := startServerFull(r, cfg) 189 | 190 | for i := 0; i < 20; i++ { 191 | _, resp2 := doGet(r, "http://127.0.0.1:60000/notify/chanws/foo") 192 | r.Equal(200, resp2.StatusCode) 193 | _, resp2 = doGet(r, "http://127.0.0.1:60000/notify/chansse/foo") 194 | r.Equal(200, resp2.StatusCode) 195 | } 196 | 197 | for i := 0; i < 10; i++ { 198 | go func() { 199 | conn, resp, err := websocket.Dial(context.Background(), "ws://127.0.0.1:60000/ws", nil) 200 | r.Nil(err) 201 | r.Equal(101, resp.StatusCode) 202 | r.NotNil(conn) 203 | conn.Close(websocket.StatusInternalError, "bye") 204 | }() 205 | 206 | go func() { 207 | conn, resp, err := websocket.Dial(context.Background(), "ws://127.0.0.1:60000/ws", nil) 208 | r.Nil(err) 209 | r.Equal(101, resp.StatusCode) 210 | r.NotNil(conn) 211 | conn.Write(context.Background(), websocket.MessageText, []byte("baz")) 212 | }() 213 | 214 | go func() { 215 | resp, err := http.Get("http://127.0.0.1:60000/sse") 216 | r.Nil(err) 217 | r.NotNil(resp) 218 | r.Equal(200, resp.StatusCode) 219 | r.NotNil(resp.Body) 220 | resp.Body.Close() 221 | }() 222 | } 223 | 224 | for i := 0; i < 20; i++ { 225 | _, resp2 := doGet(r, "http://127.0.0.1:60000/notify/chanws/foo") 226 | r.Equal(200, resp2.StatusCode) 227 | _, resp2 = doGet(r, "http://127.0.0.1:60000/notify/chansse/foo") 228 | r.Equal(200, resp2.StatusCode) 229 | } 230 | 231 | time.Sleep(time.Second) 232 | s.Stop(time.Second) 233 | } 234 | -------------------------------------------------------------------------------- /validate_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 RapidLoop, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package rapidrows_test 18 | 19 | import ( 20 | "encoding/json" 21 | "io" 22 | "os" 23 | "testing" 24 | 25 | "github.com/rapidloop/rapidrows" 26 | "github.com/stretchr/testify/require" 27 | ) 28 | 29 | const ( 30 | invalidCfgs = "_test/invalid_cfgs.jsons" 31 | warnCfgs = "_test/warn_cfgs.jsons" 32 | ) 33 | 34 | func TestValidateConfigError(t *testing.T) { 35 | r := require.New(t) 36 | 37 | f, err := os.Open(invalidCfgs) 38 | r.Nil(err) 39 | defer f.Close() 40 | 41 | dec := json.NewDecoder(f) 42 | for { 43 | var cfg rapidrows.APIServerConfig 44 | if err := dec.Decode(&cfg); err == io.EOF { 45 | break 46 | } else if err != nil { 47 | t.Fatalf("bad json in %s: %v", invalidCfgs, err) 48 | } 49 | if err := cfg.IsValid(); err == nil { 50 | t.Fatalf("invalid config passes:\n%+v\n", cfg) 51 | } else { 52 | t.Logf("error (expected): %v", err) 53 | } 54 | } 55 | } 56 | 57 | func TestValidateConfigWarn(t *testing.T) { 58 | r := require.New(t) 59 | 60 | f, err := os.Open(warnCfgs) 61 | r.Nil(err) 62 | defer f.Close() 63 | 64 | dec := json.NewDecoder(f) 65 | for { 66 | var cfg rapidrows.APIServerConfig 67 | if err := dec.Decode(&cfg); err == io.EOF { 68 | break 69 | } else if err != nil { 70 | t.Fatalf("bad json in %s: %v", warnCfgs, err) 71 | } 72 | count := 0 73 | for _, vr := range cfg.Validate() { 74 | r.True(vr.Warn, vr.Message) 75 | r.Greater(len(vr.Message), 0) 76 | t.Logf("warning (expected): %s", vr.Message) 77 | count++ 78 | } 79 | r.Greater(count, 0, "at least 1 warning was expected") 80 | } 81 | } 82 | --------------------------------------------------------------------------------