├── http--1.6--1.7.sql ├── http.control ├── .gitignore ├── http--1.4--1.5.sql ├── ci └── pg_hba.conf ├── http--1.5--1.6.sql ├── .editorconfig ├── Makefile ├── http--1.2--1.3.sql ├── http--1.1--1.2.sql ├── http--1.3--1.4.sql ├── LICENSE.md ├── META.json ├── http--1.0--1.1.sql ├── .github └── workflows │ └── ci.yml ├── http--1.7.sql ├── examples └── s3.sql ├── sql └── http.sql ├── expected └── http.out ├── README.md └── http.c /http--1.6--1.7.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /http.control: -------------------------------------------------------------------------------- 1 | default_version = '1.7' 2 | module_pathname = '$libdir/http' 3 | comment = 'HTTP client for PostgreSQL, allows web page retrieval inside the database.' 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | *.a 4 | *.pc 5 | *.dylib 6 | *.dll 7 | regression.diffs 8 | regression.out 9 | results/ 10 | tmp_check/ 11 | tmp_check_iso/ 12 | output_iso/ 13 | log/ 14 | -------------------------------------------------------------------------------- /http--1.4--1.5.sql: -------------------------------------------------------------------------------- 1 | 2 | CREATE OR REPLACE FUNCTION http_delete(uri VARCHAR, content VARCHAR, content_type VARCHAR) 3 | RETURNS http_response 4 | AS $$ SELECT @extschema@.http(('DELETE', $1, NULL, $3, $2)::@extschema@.http_request) $$ 5 | LANGUAGE 'sql'; 6 | -------------------------------------------------------------------------------- /ci/pg_hba.conf: -------------------------------------------------------------------------------- 1 | # TYPE DATABASE USER ADDRESS METHOD 2 | 3 | # "local" is for Unix domain socket connections only 4 | local all postgres trust 5 | # IPv4 local connections: 6 | host all postgres 127.0.0.1/32 trust 7 | # IPv6 local connections: 8 | host all postgres ::1/128 trust 9 | -------------------------------------------------------------------------------- /http--1.5--1.6.sql: -------------------------------------------------------------------------------- 1 | 2 | ALTER DOMAIN http_method DROP CONSTRAINT IF EXISTS http_method_check; 3 | 4 | CREATE FUNCTION text_to_bytea(data TEXT) 5 | RETURNS BYTEA 6 | AS 'MODULE_PATHNAME', 'text_to_bytea' 7 | LANGUAGE 'c' 8 | IMMUTABLE STRICT; 9 | 10 | CREATE FUNCTION bytea_to_text(data BYTEA) 11 | RETURNS TEXT 12 | AS 'MODULE_PATHNAME', 'bytea_to_text' 13 | LANGUAGE 'c' 14 | IMMUTABLE STRICT; 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # these are the defaults 7 | [*] 8 | charset = utf-8 9 | end_of_line = lf 10 | trim_trailing_whitespace = true 11 | insert_final_newline = true 12 | 13 | # C files want tab indentation 14 | [*.{c,h}] 15 | indent_style = tab 16 | 17 | # YAML files want space indentation 18 | [*.{yml}] 19 | indent_style = space 20 | indent_size = 4 21 | 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | MODULE_big = http 3 | OBJS = http.o 4 | EXTENSION = http 5 | 6 | DATA = $(wildcard *.sql) 7 | 8 | REGRESS = http 9 | EXTRA_CLEAN = 10 | 11 | CURL_CONFIG = curl-config 12 | PG_CONFIG = pg_config 13 | 14 | CFLAGS += $(shell $(CURL_CONFIG) --cflags) 15 | LIBS += $(shell $(CURL_CONFIG) --libs) 16 | SHLIB_LINK := $(LIBS) 17 | 18 | ifdef DEBUG 19 | COPT += -O0 -Werror -g 20 | endif 21 | 22 | PGXS := $(shell $(PG_CONFIG) --pgxs) 23 | include $(PGXS) 24 | 25 | -------------------------------------------------------------------------------- /http--1.2--1.3.sql: -------------------------------------------------------------------------------- 1 | ALTER DOMAIN http_method DROP CONSTRAINT http_method_check; 2 | ALTER DOMAIN http_method ADD CHECK ( 3 | VALUE ILIKE 'get' OR 4 | VALUE ILIKE 'post' OR 5 | VALUE ILIKE 'put' OR 6 | VALUE ILIKE 'delete' OR 7 | VALUE ILIKE 'patch' OR 8 | VALUE ILIKE 'head' 9 | ); 10 | 11 | CREATE OR REPLACE FUNCTION http_patch(uri VARCHAR, content VARCHAR, content_type VARCHAR) 12 | RETURNS http_response 13 | AS $$ SELECT @extschema@.http(('PATCH', $1, NULL, $3, $2)::http_request) $$ 14 | LANGUAGE 'sql'; 15 | -------------------------------------------------------------------------------- /http--1.1--1.2.sql: -------------------------------------------------------------------------------- 1 | 2 | ALTER DOMAIN http_method DROP CONSTRAINT http_method_check; 3 | ALTER DOMAIN http_method ADD CHECK ( 4 | VALUE ILIKE 'get' OR 5 | VALUE ILIKE 'post' OR 6 | VALUE ILIKE 'put' OR 7 | VALUE ILIKE 'delete' OR 8 | VALUE ILIKE 'head' 9 | ); 10 | 11 | CREATE OR REPLACE FUNCTION http_head(uri VARCHAR) 12 | RETURNS http_response 13 | AS $$ SELECT @extschema@.http(('HEAD', $1, NULL, NULL, NULL)::http_request) $$ 14 | LANGUAGE 'sql'; 15 | 16 | CREATE OR REPLACE FUNCTION http_set_curlopt (curlopt VARCHAR, value VARCHAR) 17 | RETURNS boolean 18 | AS 'MODULE_PATHNAME', 'http_set_curlopt' 19 | LANGUAGE 'c'; 20 | 21 | CREATE OR REPLACE FUNCTION http_reset_curlopt () 22 | RETURNS boolean 23 | AS 'MODULE_PATHNAME', 'http_reset_curlopt' 24 | LANGUAGE 'c'; 25 | -------------------------------------------------------------------------------- /http--1.3--1.4.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION http_list_curlopt () 2 | RETURNS TABLE(curlopt text, value text) 3 | AS 'MODULE_PATHNAME', 'http_list_curlopt' 4 | LANGUAGE 'c'; 5 | 6 | CREATE OR REPLACE FUNCTION urlencode(string BYTEA) 7 | RETURNS TEXT 8 | AS 'MODULE_PATHNAME' 9 | LANGUAGE 'c' 10 | IMMUTABLE STRICT; 11 | 12 | CREATE OR REPLACE FUNCTION urlencode(data JSONB) 13 | RETURNS TEXT 14 | AS 'MODULE_PATHNAME' 15 | LANGUAGE 'c' 16 | IMMUTABLE STRICT; 17 | 18 | CREATE OR REPLACE FUNCTION http_get(uri VARCHAR, data JSONB) 19 | RETURNS http_response 20 | AS $$ SELECT @extschema@.http(('GET', $1 || '?' || @extschema@.urlencode($2), NULL, NULL, NULL)::http_request) $$ 21 | LANGUAGE 'sql'; 22 | 23 | CREATE OR REPLACE FUNCTION http_post(uri VARCHAR, data JSONB) 24 | RETURNS http_response 25 | AS $$ SELECT @extschema@.http(('POST', $1, NULL, 'application/x-www-form-urlencoded', @extschema@.urlencode($2))::http_request) $$ 26 | LANGUAGE 'sql'; 27 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (C) 2025 Paul Ramsey 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /META.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "http", 3 | "abstract": "HTTP client for PostgreSQL", 4 | "description": "HTTP allows you to get the content of a web page in a SQL function call.", 5 | "version": "1.7.1", 6 | "maintainer": [ 7 | "Paul Ramsey " 8 | ], 9 | "license": { 10 | "mit": "http://en.wikipedia.org/wiki/MIT_License" 11 | }, 12 | "prereqs": { 13 | "runtime": { 14 | "requires": { 15 | "PostgreSQL": "9.1.0" 16 | }, 17 | "recommends": { 18 | "PostgreSQL": "9.1.3" 19 | } 20 | } 21 | }, 22 | "provides": { 23 | "http": { 24 | "file": "http--1.7.sql", 25 | "docfile": "README.md", 26 | "version": "1.7.1", 27 | "abstract": "HTTP client for PostgreSQL" 28 | } 29 | }, 30 | "resources": { 31 | "homepage": "https://github.com/pramsey/pgsql-http/", 32 | "bugtracker": { 33 | "web": "https://github.com/pramsey/pgsql-http/issues" 34 | }, 35 | "repository": { 36 | "url": "https://github.com/pramsey/pgsql-http.git", 37 | "web": "https://github.com/pramsey/pgsql-http/", 38 | "type": "git" 39 | } 40 | }, 41 | "generated_by": "Paul Ramsey", 42 | "meta-spec": { 43 | "version": "1.0.0", 44 | "url": "http://pgxn.org/meta/spec.txt" 45 | }, 46 | "tags": [ 47 | "http", 48 | "curl", 49 | "web" 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /http--1.0--1.1.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Remove old 1.0 functions 3 | DROP FUNCTION http_get(varchar, varchar); 4 | DROP FUNCTION http_post(varchar, varchar, varchar, varchar); 5 | 6 | CREATE DOMAIN http_method AS text 7 | CHECK ( 8 | VALUE ILIKE 'get' OR 9 | VALUE ILIKE 'post' OR 10 | VALUE ILIKE 'put' OR 11 | VALUE ILIKE 'delete' 12 | ); 13 | 14 | CREATE DOMAIN content_type AS text 15 | CHECK ( 16 | VALUE ~ '^\S+\/\S+' 17 | ); 18 | 19 | CREATE TYPE http_header AS ( 20 | field VARCHAR, 21 | value VARCHAR 22 | ); 23 | 24 | CREATE TYPE http_request AS ( 25 | method http_method, 26 | uri VARCHAR, 27 | headers http_header[], 28 | content_type VARCHAR, 29 | content VARCHAR 30 | ); 31 | 32 | ALTER TYPE http_response ALTER ATTRIBUTE headers TYPE http_header[]; 33 | 34 | CREATE OR REPLACE FUNCTION http_header (field VARCHAR, value VARCHAR) 35 | RETURNS http_header 36 | AS $$ SELECT $1, $2 $$ 37 | LANGUAGE 'sql'; 38 | 39 | CREATE OR REPLACE FUNCTION http(request @extschema@.http_request) 40 | RETURNS http_response 41 | AS 'MODULE_PATHNAME', 'http_request' 42 | LANGUAGE 'c'; 43 | 44 | CREATE OR REPLACE FUNCTION http_get(uri VARCHAR) 45 | RETURNS http_response 46 | AS $$ SELECT @extschema@.http(('GET', $1, NULL, NULL, NULL)::http_request) $$ 47 | LANGUAGE 'sql'; 48 | 49 | CREATE OR REPLACE FUNCTION http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR) 50 | RETURNS http_response 51 | AS $$ SELECT @extschema@.http(('POST', $1, NULL, $3, $2)::http_request) $$ 52 | LANGUAGE 'sql'; 53 | 54 | CREATE OR REPLACE FUNCTION http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR) 55 | RETURNS http_response 56 | AS $$ SELECT @extschema@.http(('PUT', $1, NULL, $3, $2)::http_request) $$ 57 | LANGUAGE 'sql'; 58 | 59 | CREATE OR REPLACE FUNCTION http_delete(uri VARCHAR) 60 | RETURNS http_response 61 | AS $$ SELECT @extschema@.http(('DELETE', $1, NULL, NULL, NULL)::http_request) $$ 62 | LANGUAGE 'sql'; 63 | 64 | CREATE OR REPLACE FUNCTION urlencode(string VARCHAR) 65 | RETURNS TEXT 66 | AS 'MODULE_PATHNAME' 67 | LANGUAGE 'c' 68 | IMMUTABLE STRICT; 69 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # GitHub Actions for pgSQL HTTP 2 | # 3 | # Paul Ramsey 4 | 5 | name: "CI" 6 | on: [push, pull_request] 7 | 8 | jobs: 9 | linux: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | name: "CI" 14 | strategy: 15 | fail-fast: true 16 | matrix: 17 | ci: 18 | - { PGVER: 13 } 19 | - { PGVER: 14 } 20 | - { PGVER: 15 } 21 | - { PGVER: 16 } 22 | - { PGVER: 17 } 23 | - { PGVER: 18 } 24 | 25 | steps: 26 | 27 | - name: 'Check Out' 28 | uses: actions/checkout@v4 29 | 30 | - name: 'Raise Priority for apt.postgresql.org' 31 | run: | 32 | cat << EOF >> ./pgdg.pref 33 | Package: * 34 | Pin: release o=apt.postgresql.org 35 | Pin-Priority: 600 36 | EOF 37 | sudo mv ./pgdg.pref /etc/apt/preferences.d/ 38 | sudo apt update 39 | 40 | - name: 'Install PostgreSQL' 41 | run: | 42 | sudo apt-get purge postgresql-* 43 | sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg-snapshot main ${{ matrix.ci.PGVER }}" > /etc/apt/sources.list.d/pgdg.list' 44 | curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null 45 | sudo apt-get update 46 | sudo apt-get -y install postgresql-${{ matrix.ci.PGVER }} postgresql-server-dev-${{ matrix.ci.PGVER }} postgresql-client-${{ matrix.ci.PGVER }} 47 | 48 | - name: 'Install Curl' 49 | run: | 50 | sudo apt-get -y install libcurl4-gnutls-dev 51 | 52 | - name: 'Start PostgreSQL' 53 | run: | 54 | export PGVER=${{ matrix.ci.PGVER }} 55 | export PGDATA=/var/lib/postgresql/$PGVER/main 56 | export PGETC=/etc/postgresql/$PGVER/main 57 | export PGBIN=/usr/lib/postgresql/$PGVER/bin 58 | sudo chmod -R 755 /home/`whoami` 59 | sudo cp ./ci/pg_hba.conf $PGETC/pg_hba.conf 60 | sudo systemctl stop postgresql 61 | sudo pg_ctlcluster $PGVER main start 62 | sudo pg_lsclusters 63 | 64 | - name: 'Start HttpBin Docker' 65 | run: | 66 | docker run -d -p 9080:80 kennethreitz/httpbin 67 | 68 | - name: 'Build & Test' 69 | run: | 70 | export PATH=/usr/lib/postgresql/${{ matrix.ci.PGVER }}/bin/:$PATH 71 | export PG_CONFIG=/usr/lib/postgresql/${{ matrix.ci.PGVER }}/bin/pg_config 72 | export PG_CFLAGS=-Werror 73 | make 74 | sudo -E make PG_CONFIG=$PG_CONFIG install 75 | PGUSER=postgres make installcheck || (cat regression.diffs && /bin/false) 76 | 77 | -------------------------------------------------------------------------------- /http--1.7.sql: -------------------------------------------------------------------------------- 1 | -- complain if script is sourced in psql, rather than via CREATE EXTENSION 2 | \echo Use "CREATE EXTENSION http" to load this file. \quit 3 | CREATE DOMAIN http_method AS text; 4 | CREATE DOMAIN content_type AS text 5 | CHECK ( 6 | VALUE ~ '^\S+\/\S+' 7 | ); 8 | 9 | CREATE TYPE http_header AS ( 10 | field VARCHAR, 11 | value VARCHAR 12 | ); 13 | 14 | CREATE TYPE http_response AS ( 15 | status INTEGER, 16 | content_type VARCHAR, 17 | headers http_header[], 18 | content VARCHAR 19 | ); 20 | 21 | CREATE TYPE http_request AS ( 22 | method http_method, 23 | uri VARCHAR, 24 | headers http_header[], 25 | content_type VARCHAR, 26 | content VARCHAR 27 | ); 28 | 29 | CREATE FUNCTION http_set_curlopt (curlopt VARCHAR, value VARCHAR) 30 | RETURNS boolean 31 | AS 'MODULE_PATHNAME', 'http_set_curlopt' 32 | LANGUAGE 'c'; 33 | 34 | CREATE FUNCTION http_reset_curlopt () 35 | RETURNS boolean 36 | AS 'MODULE_PATHNAME', 'http_reset_curlopt' 37 | LANGUAGE 'c'; 38 | 39 | CREATE FUNCTION http_list_curlopt () 40 | RETURNS TABLE(curlopt text, value text) 41 | AS 'MODULE_PATHNAME', 'http_list_curlopt' 42 | LANGUAGE 'c'; 43 | 44 | CREATE FUNCTION http_header (field VARCHAR, value VARCHAR) 45 | RETURNS http_header 46 | AS $$ SELECT $1, $2 $$ 47 | LANGUAGE 'sql'; 48 | 49 | CREATE FUNCTION http(request @extschema@.http_request) 50 | RETURNS http_response 51 | AS 'MODULE_PATHNAME', 'http_request' 52 | LANGUAGE 'c'; 53 | 54 | CREATE FUNCTION http_get(uri VARCHAR) 55 | RETURNS http_response 56 | AS $$ SELECT @extschema@.http(('GET', $1, NULL, NULL, NULL)::@extschema@.http_request) $$ 57 | LANGUAGE 'sql'; 58 | 59 | CREATE FUNCTION http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR) 60 | RETURNS http_response 61 | AS $$ SELECT @extschema@.http(('POST', $1, NULL, $3, $2)::@extschema@.http_request) $$ 62 | LANGUAGE 'sql'; 63 | 64 | CREATE FUNCTION http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR) 65 | RETURNS http_response 66 | AS $$ SELECT @extschema@.http(('PUT', $1, NULL, $3, $2)::@extschema@.http_request) $$ 67 | LANGUAGE 'sql'; 68 | 69 | CREATE FUNCTION http_patch(uri VARCHAR, content VARCHAR, content_type VARCHAR) 70 | RETURNS http_response 71 | AS $$ SELECT @extschema@.http(('PATCH', $1, NULL, $3, $2)::@extschema@.http_request) $$ 72 | LANGUAGE 'sql'; 73 | 74 | CREATE FUNCTION http_delete(uri VARCHAR) 75 | RETURNS http_response 76 | AS $$ SELECT @extschema@.http(('DELETE', $1, NULL, NULL, NULL)::@extschema@.http_request) $$ 77 | LANGUAGE 'sql'; 78 | 79 | CREATE FUNCTION http_delete(uri VARCHAR, content VARCHAR, content_type VARCHAR) 80 | RETURNS http_response 81 | AS $$ SELECT @extschema@.http(('DELETE', $1, NULL, $3, $2)::@extschema@.http_request) $$ 82 | LANGUAGE 'sql'; 83 | 84 | CREATE FUNCTION http_head(uri VARCHAR) 85 | RETURNS http_response 86 | AS $$ SELECT @extschema@.http(('HEAD', $1, NULL, NULL, NULL)::@extschema@.http_request) $$ 87 | LANGUAGE 'sql'; 88 | 89 | CREATE FUNCTION urlencode(string VARCHAR) 90 | RETURNS TEXT 91 | AS 'MODULE_PATHNAME' 92 | LANGUAGE 'c' 93 | IMMUTABLE STRICT; 94 | 95 | CREATE FUNCTION urlencode(string BYTEA) 96 | RETURNS TEXT 97 | AS 'MODULE_PATHNAME' 98 | LANGUAGE 'c' 99 | IMMUTABLE STRICT; 100 | 101 | CREATE FUNCTION urlencode(data JSONB) 102 | RETURNS TEXT 103 | AS 'MODULE_PATHNAME', 'urlencode_jsonb' 104 | LANGUAGE 'c' 105 | IMMUTABLE STRICT; 106 | 107 | CREATE FUNCTION http_get(uri VARCHAR, data JSONB) 108 | RETURNS http_response 109 | AS $$ 110 | SELECT @extschema@.http(('GET', $1 || '?' || @extschema@.urlencode($2), NULL, NULL, NULL)::@extschema@.http_request) 111 | $$ 112 | LANGUAGE 'sql'; 113 | 114 | CREATE FUNCTION http_post(uri VARCHAR, data JSONB) 115 | RETURNS http_response 116 | AS $$ 117 | SELECT @extschema@.http(('POST', $1, NULL, 'application/x-www-form-urlencoded', @extschema@.urlencode($2))::@extschema@.http_request) 118 | $$ 119 | LANGUAGE 'sql'; 120 | 121 | CREATE FUNCTION text_to_bytea(data TEXT) 122 | RETURNS BYTEA 123 | AS 'MODULE_PATHNAME', 'text_to_bytea' 124 | LANGUAGE 'c' 125 | IMMUTABLE STRICT; 126 | 127 | CREATE FUNCTION bytea_to_text(data BYTEA) 128 | RETURNS TEXT 129 | AS 'MODULE_PATHNAME', 'bytea_to_text' 130 | LANGUAGE 'c' 131 | IMMUTABLE STRICT; 132 | 133 | CREATE FUNCTION http_headers(VARIADIC args text[]) 134 | RETURNS http_header[] AS $$ 135 | DECLARE 136 | headers http_header[]; 137 | i int; 138 | BEGIN 139 | -- Ensure the number of arguments is even 140 | IF array_length(args, 1) % 2 <> 0 THEN 141 | RAISE EXCEPTION 'Arguments must be provided in key-value pairs'; 142 | END IF; 143 | 144 | -- Iterate over the arguments two at a time 145 | FOR i IN 1..array_length(args, 1) BY 2 LOOP 146 | headers := array_append(headers, http_header(args[i], args[i+1])); 147 | END LOOP; 148 | 149 | RETURN headers; 150 | END; 151 | $$ 152 | LANGUAGE 'plpgsql' 153 | IMMUTABLE STRICT; 154 | -------------------------------------------------------------------------------- /examples/s3.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- s3_request 3 | -- 4 | -- Installation: 5 | -- 6 | -- CREATE EXTENSION pgcrypto; 7 | -- CREATE EXTENSION http; 8 | -- 9 | -- Utility function to take S3 object and access keys and create 10 | -- a signed HTTP request using the AWS4 signing scheme. 11 | -- https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html 12 | -- 13 | -- Various pieces of the request are gathered into strings bundled together 14 | -- and ultimately signed with the s3 secret key. 15 | -- 16 | -- Example: 17 | -- 18 | -- https://cleverelephant-west-1.s3.amazonaws.com/META.json 19 | -- 20 | -- SELECT * FROM s3_request( 21 | -- 'your_s3_access_key', -- access 22 | -- 'your_s3_secret_key', -- secret 23 | -- 'us-west-1', -- region 24 | -- 'cleverelephant-west-1', -- bucket 25 | -- 'META.json', -- object name 26 | -- ); 27 | -- 28 | -- 29 | -- Create and delete objects too! 30 | -- 31 | -- SELECT * FROM s3_request( 32 | -- 'your_s3_access_key', -- access 33 | -- 'your_s3_secret_key', -- secret 34 | -- 'us-west-1', -- region 35 | -- 'cleverelephant-west-1', -- bucket 36 | -- 'testfile.txt', -- object name 37 | -- 'PUT', -- http method 38 | -- 'this is a test' -- payload 39 | -- 'text/plain' -- payload mime type 40 | -- ); 41 | -- 42 | -- SELECT * FROM s3_request( 43 | -- 'your_s3_access_key', -- access 44 | -- 'your_s3_secret_key', -- secret 45 | -- 'us-west-1', -- region 46 | -- 'cleverelephant-west-1', -- bucket 47 | -- 'testfile.txt', -- object name 48 | -- ); 49 | -- 50 | -- SELECT * FROM s3_request( 51 | -- 'your_s3_access_key', -- access 52 | -- 'your_s3_secret_key', -- secret 53 | -- 'us-west-1', -- region 54 | -- 'cleverelephant-west-1', -- bucket 55 | -- 'testfile.txt', -- object name 56 | -- 'DELETE' -- http method 57 | -- ); 58 | -- 59 | 60 | CREATE OR REPLACE FUNCTION s3_request( 61 | access_key TEXT, 62 | secret_key TEXT, 63 | region TEXT, 64 | bucket TEXT, 65 | object_key TEXT, 66 | http_method TEXT DEFAULT 'GET', 67 | object_payload TEXT DEFAULT NULL, 68 | object_mimetype TEXT DEFAULT 'text/plain' 69 | ) RETURNS http_response AS $$ 70 | DECLARE 71 | host TEXT := bucket || '.s3.' || region || '.amazonaws.com'; 72 | endpoint TEXT := 'https://' || host || '/' || object_key; 73 | canonical_uri TEXT := '/' || object_key; 74 | canonical_querystring TEXT := ''; 75 | signed_headers TEXT := 'host;x-amz-content-sha256;x-amz-date'; 76 | service TEXT := 's3'; 77 | 78 | now TIMESTAMP := now() AT TIME ZONE 'UTC'; 79 | amz_date TEXT := to_char(now, 'YYYYMMDD"T"HH24MISS"Z"'); 80 | date_stamp TEXT := to_char(now, 'YYYYMMDD'); 81 | 82 | -- Must use this magic hash if the payload is empty 83 | payload_hash TEXT := 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'; 84 | 85 | canonical_headers TEXT; 86 | canonical_request TEXT; 87 | string_to_sign TEXT; 88 | credential_scope TEXT; 89 | date_key BYTEA; 90 | date_region_key BYTEA; 91 | date_region_service_key BYTEA; 92 | signing_key BYTEA; 93 | signature TEXT; 94 | authorization_header TEXT; 95 | canonical_request_digest TEXT; 96 | request http_request; 97 | BEGIN 98 | 99 | http_method := upper(http_method); 100 | object_mimetype := lower(object_mimetype); 101 | 102 | IF object_payload IS NOT NULL 103 | THEN 104 | payload_hash := encode(digest(convert_to(object_payload, 'UTF8'), 'sha256'), 'hex'); 105 | END IF; 106 | 107 | -- Construct the canonical headers 108 | canonical_headers := 'host:' || host || E'\n' 109 | || 'x-amz-content-sha256:' || payload_hash || E'\n' 110 | || 'x-amz-date:' || amz_date || E'\n'; 111 | 112 | -- Signed headers must be in alphabetical order 113 | -- so content-type goes first 114 | IF object_payload IS NOT NULL 115 | THEN 116 | canonical_headers := 'content-type:' || object_mimetype || E'\n' || canonical_headers; 117 | signed_headers := 'content-type;' || signed_headers; 118 | END IF; 119 | 120 | -- Create the canonical request 121 | canonical_request := http_method || E'\n' || 122 | canonical_uri || E'\n' || 123 | canonical_querystring || E'\n' || 124 | canonical_headers || E'\n' || 125 | signed_headers || E'\n' || 126 | payload_hash; 127 | 128 | -- Define the credential scope 129 | credential_scope := date_stamp || '/' || region || '/' || service || '/aws4_request'; 130 | 131 | -- Get sha256 hash of request 132 | canonical_request_digest := encode(digest(canonical_request, 'sha256'), 'hex'); 133 | 134 | -- Create the string to sign 135 | string_to_sign := 'AWS4-HMAC-SHA256' || E'\n' || 136 | amz_date || E'\n' || 137 | credential_scope || E'\n' || 138 | canonical_request_digest; 139 | 140 | -- 141 | -- Signature of pgcrypto function is hmac(payload, secret, algo) 142 | -- Each piece of the signing key is bundled together with the 143 | -- previous piece, starting with the S3 secret key. 144 | -- 145 | date_key := hmac(convert_to(date_stamp, 'UTF8'), convert_to('AWS4' || secret_key, 'UTF8'), 'sha256'); 146 | date_region_key := hmac(convert_to(region, 'UTF8'), date_key, 'sha256'); 147 | date_region_service_key := hmac(convert_to(service, 'UTF8'), date_region_key, 'sha256'); 148 | signing_key := hmac(convert_to('aws4_request','UTF8'), date_region_service_key, 'sha256'); 149 | 150 | -- Compute the signature 151 | signature := encode(hmac(convert_to(string_to_sign, 'UTF8'), signing_key, 'sha256'), 'hex'); 152 | 153 | -- Construct the Authorization header 154 | authorization_header := 'AWS4-HMAC-SHA256 Credential=' || access_key || '/' || credential_scope || 155 | ', SignedHeaders=' || signed_headers || 156 | ', Signature=' || signature; 157 | 158 | 159 | -- Perform the HTTP request 160 | request := ( 161 | http_method, 162 | endpoint, 163 | http_headers('Authorization', authorization_header, 164 | 'x-amz-content-sha256', payload_hash, 165 | 'x-amz-date', amz_date, 166 | 'host', host), 167 | object_mimetype, 168 | object_payload 169 | )::http_request; 170 | 171 | -- Getting the canonical request and payload strings perfectly 172 | -- formatted is an important step so debugging here in case 173 | -- S3 rejects signed request 174 | RAISE DEBUG 's3_request, payload_hash: %', payload_hash; 175 | RAISE DEBUG 's3_request, canonical_request: %', canonical_request; 176 | RAISE DEBUG 's3_request, string_to_sign: %', string_to_sign; 177 | RAISE DEBUG 's3_request, request %', request; 178 | 179 | RETURN http(request); 180 | 181 | END; 182 | $$ LANGUAGE 'plpgsql' 183 | VOLATILE; 184 | -------------------------------------------------------------------------------- /sql/http.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION http; 2 | SET http.server_host = 'http://localhost:9080'; 3 | set http.timeout_msec = 10000; 4 | SELECT http_set_curlopt('CURLOPT_TIMEOUT', '10'); 5 | -- if local server not up use global one 6 | DO language plpgsql $$ 7 | BEGIN 8 | BEGIN 9 | PERFORM http_get(current_setting('http.server_host') || '/status/202'); 10 | EXCEPTION WHEN OTHERS THEN 11 | SET http.server_host = 'http://httpbin.org'; 12 | END; 13 | END; 14 | $$; 15 | 16 | -- Status code 17 | SELECT status 18 | FROM http_get(current_setting('http.server_host') || '/status/202'); 19 | 20 | -- Headers 21 | SELECT lower(field) AS field, value 22 | FROM ( 23 | SELECT (unnest(headers)).* 24 | FROM http_get(current_setting('http.server_host') || '/response-headers?Abcde=abcde') 25 | ) a 26 | WHERE field ILIKE 'Abcde'; 27 | 28 | -- GET 29 | SELECT status, 30 | content::json->'args'->>'foo' AS args, 31 | content::json->>'method' AS method 32 | FROM http_get(current_setting('http.server_host') || '/anything?foo=bar'); 33 | 34 | -- GET with data 35 | SELECT status, 36 | content::json->'args'->>'this' AS args, 37 | replace(content::json->>'url',current_setting('http.server_host'),'') AS path, 38 | content::json->>'method' AS method 39 | FROM http_get(current_setting('http.server_host') || '/anything', jsonb_build_object('this', 'that')); 40 | 41 | -- GET with data 42 | SELECT status, 43 | content::json->>'args' as args, 44 | (content::json)->>'data' as data, 45 | content::json->>'method' as method 46 | FROM http(('GET', current_setting('http.server_host') || '/anything', NULL, 'application/json', '{"search": "toto"}')); 47 | 48 | -- DELETE 49 | SELECT status, 50 | content::json->'args'->>'foo' AS args, 51 | replace(content::json->>'url',current_setting('http.server_host'),'') AS path, 52 | content::json->>'method' AS method 53 | FROM http_delete(current_setting('http.server_host') || '/anything?foo=bar'); 54 | 55 | -- DELETE with payload 56 | SELECT status, 57 | content::json->'args'->>'foo' AS args, 58 | replace(content::json->>'url',current_setting('http.server_host'),'') AS path, 59 | content::json->>'method' AS method, 60 | content::json->>'data' AS data 61 | FROM http_delete(current_setting('http.server_host') || '/anything?foo=bar', 'payload', 'text/plain'); 62 | 63 | -- PUT 64 | SELECT status, 65 | content::json->>'data' AS data, 66 | content::json->'args'->>'foo' AS args, 67 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 68 | content::json->>'method' AS method 69 | FROM http_put(current_setting('http.server_host') || '/anything?foo=bar','payload','text/plain'); 70 | 71 | -- PATCH 72 | SELECT status, 73 | content::json->>'data' AS data, 74 | content::json->'args'->>'foo' AS args, 75 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 76 | content::json->>'method' AS method 77 | FROM http_patch(current_setting('http.server_host') || '/anything?foo=bar','{"this":"that"}','application/json'); 78 | 79 | -- POST 80 | SELECT status, 81 | content::json->>'data' AS data, 82 | content::json->'args'->>'foo' AS args, 83 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 84 | content::json->>'method' AS method 85 | FROM http_post(current_setting('http.server_host') || '/anything?foo=bar','payload','text/plain'); 86 | 87 | -- POST with json data 88 | SELECT status, 89 | content::json->'form'->>'this' AS args, 90 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 91 | content::json->>'method' AS method 92 | FROM http_post(current_setting('http.server_host') || '/anything', jsonb_build_object('this', 'that')); 93 | 94 | -- POST with data 95 | SELECT status, 96 | content::json->'form'->>'key1' AS key1, 97 | content::json->'form'->>'key2' AS key2, 98 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 99 | content::json->>'method' AS method 100 | FROM http_post(current_setting('http.server_host') || '/anything', 'key1=value1&key2=value2','application/x-www-form-urlencoded'); 101 | 102 | -- HEAD 103 | SELECT lower(field) AS field, value 104 | FROM ( 105 | SELECT (unnest(headers)).* 106 | FROM http_head(current_setting('http.server_host') || '/response-headers?Abcde=abcde') 107 | ) a 108 | WHERE field ILIKE 'Abcde'; 109 | 110 | -- Follow redirect 111 | SELECT status, 112 | replace((content::json)->>'url', current_setting('http.server_host'),'') AS path 113 | FROM http_get(current_setting('http.server_host') || '/redirect-to?url=get'); 114 | 115 | -- Request image 116 | WITH 117 | http AS ( 118 | SELECT * FROM http_get(current_setting('http.server_host') || '/image/png') 119 | ), 120 | headers AS ( 121 | SELECT (unnest(headers)).* FROM http 122 | ) 123 | SELECT 124 | http.content_type, 125 | length(text_to_bytea(http.content)) AS length_binary 126 | FROM http, headers 127 | WHERE field ilike 'Content-Type'; 128 | 129 | -- Alter options and and reset them and throw errors 130 | SELECT http_set_curlopt('CURLOPT_PROXY', '127.0.0.1'); 131 | -- Error because proxy is not there 132 | DO $$ 133 | BEGIN 134 | SELECT status FROM http_get(current_setting('http.server_host') || '/status/555'); 135 | EXCEPTION 136 | WHEN OTHERS THEN 137 | RAISE WARNING 'Failed to connect'; 138 | END; 139 | $$; 140 | -- Still an error 141 | DO $$ 142 | BEGIN 143 | SELECT status FROM http_get(current_setting('http.server_host') || '/status/555'); 144 | EXCEPTION 145 | WHEN OTHERS THEN 146 | RAISE WARNING 'Failed to connect'; 147 | END; 148 | $$; 149 | -- Reset options 150 | SELECT http_reset_curlopt(); 151 | -- Now it should work 152 | SELECT status FROM http_get(current_setting('http.server_host') || '/status/555'); 153 | 154 | -- Alter the default timeout and then run a query that is longer than 155 | -- the default (5s), but shorter than the new timeout 156 | SELECT http_set_curlopt('CURLOPT_TIMEOUT_MS', '10000'); 157 | SELECT status FROM http_get(current_setting('http.server_host') || '/delay/7'); 158 | 159 | -- Test new GUC feature 160 | SET http.CURLOPT_TIMEOUT_MS = '10'; 161 | -- should fail 162 | -- Still an error 163 | DO $$ 164 | BEGIN 165 | SELECT status FROM http_get(current_setting('http.server_host') || '/delay/7'); 166 | EXCEPTION 167 | WHEN OTHERS THEN 168 | RAISE WARNING 'Failed to connect'; 169 | END; 170 | $$; 171 | 172 | SET http.CURLOPT_TIMEOUT_MS = '10000'; 173 | --should pass 174 | SELECT status FROM http_get(current_setting('http.server_host') || '/delay/7'); 175 | 176 | -- SET to bogus file 177 | SET http.CURLOPT_CAINFO = '/path/to/somebundle.crt'; 178 | 179 | -- should fail 180 | DO $$ 181 | BEGIN 182 | SELECT status FROM http_get('https://postgis.net'); 183 | EXCEPTION 184 | WHEN OTHERS THEN 185 | RAISE WARNING 'Invalid cert file'; 186 | END; 187 | $$; 188 | 189 | -- set to ignore cert 190 | SET http.CURLOPT_SSL_VERIFYPEER = '0'; 191 | 192 | -- should pass 193 | SELECT status FROM http_get('https://postgis.net'); 194 | 195 | SHOW http.CURLOPT_CAINFO; 196 | 197 | -- reset it 198 | RESET http.CURLOPT_CAINFO; 199 | 200 | SELECT status FROM http_get(current_setting('http.server_host') || '/delay/7'); 201 | 202 | -- Check that statement interruption works 203 | SET statement_timeout = 200; 204 | CREATE TEMPORARY TABLE timer AS 205 | SELECT now() AS start; 206 | SELECT * 207 | FROM http_get(current_setting('http.server_host') || '/delay/7'); 208 | SELECT round(extract(epoch FROM now() - start) * 10) AS m 209 | FROM timer; 210 | DROP TABLE timer; 211 | -------------------------------------------------------------------------------- /expected/http.out: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION http; 2 | SET http.server_host = 'http://localhost:9080'; 3 | set http.timeout_msec = 10000; 4 | SELECT http_set_curlopt('CURLOPT_TIMEOUT', '10'); 5 | http_set_curlopt 6 | ------------------ 7 | t 8 | (1 row) 9 | 10 | -- if local server not up use global one 11 | DO language plpgsql $$ 12 | BEGIN 13 | BEGIN 14 | PERFORM http_get(current_setting('http.server_host') || '/status/202'); 15 | EXCEPTION WHEN OTHERS THEN 16 | SET http.server_host = 'http://httpbin.org'; 17 | END; 18 | END; 19 | $$; 20 | -- Status code 21 | SELECT status 22 | FROM http_get(current_setting('http.server_host') || '/status/202'); 23 | status 24 | -------- 25 | 202 26 | (1 row) 27 | 28 | -- Headers 29 | SELECT lower(field) AS field, value 30 | FROM ( 31 | SELECT (unnest(headers)).* 32 | FROM http_get(current_setting('http.server_host') || '/response-headers?Abcde=abcde') 33 | ) a 34 | WHERE field ILIKE 'Abcde'; 35 | field | value 36 | -------+------- 37 | abcde | abcde 38 | (1 row) 39 | 40 | -- GET 41 | SELECT status, 42 | content::json->'args'->>'foo' AS args, 43 | content::json->>'method' AS method 44 | FROM http_get(current_setting('http.server_host') || '/anything?foo=bar'); 45 | status | args | method 46 | --------+------+-------- 47 | 200 | bar | GET 48 | (1 row) 49 | 50 | -- GET with data 51 | SELECT status, 52 | content::json->'args'->>'this' AS args, 53 | replace(content::json->>'url',current_setting('http.server_host'),'') AS path, 54 | content::json->>'method' AS method 55 | FROM http_get(current_setting('http.server_host') || '/anything', jsonb_build_object('this', 'that')); 56 | status | args | path | method 57 | --------+------+---------------------+-------- 58 | 200 | that | /anything?this=that | GET 59 | (1 row) 60 | 61 | -- GET with data 62 | SELECT status, 63 | content::json->>'args' as args, 64 | (content::json)->>'data' as data, 65 | content::json->>'method' as method 66 | FROM http(('GET', current_setting('http.server_host') || '/anything', NULL, 'application/json', '{"search": "toto"}')); 67 | status | args | data | method 68 | --------+------+--------------------+-------- 69 | 200 | {} | {"search": "toto"} | GET 70 | (1 row) 71 | 72 | -- DELETE 73 | SELECT status, 74 | content::json->'args'->>'foo' AS args, 75 | replace(content::json->>'url',current_setting('http.server_host'),'') AS path, 76 | content::json->>'method' AS method 77 | FROM http_delete(current_setting('http.server_host') || '/anything?foo=bar'); 78 | status | args | path | method 79 | --------+------+-------------------+-------- 80 | 200 | bar | /anything?foo=bar | DELETE 81 | (1 row) 82 | 83 | -- DELETE with payload 84 | SELECT status, 85 | content::json->'args'->>'foo' AS args, 86 | replace(content::json->>'url',current_setting('http.server_host'),'') AS path, 87 | content::json->>'method' AS method, 88 | content::json->>'data' AS data 89 | FROM http_delete(current_setting('http.server_host') || '/anything?foo=bar', 'payload', 'text/plain'); 90 | status | args | path | method | data 91 | --------+------+-------------------+--------+--------- 92 | 200 | bar | /anything?foo=bar | DELETE | payload 93 | (1 row) 94 | 95 | -- PUT 96 | SELECT status, 97 | content::json->>'data' AS data, 98 | content::json->'args'->>'foo' AS args, 99 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 100 | content::json->>'method' AS method 101 | FROM http_put(current_setting('http.server_host') || '/anything?foo=bar','payload','text/plain'); 102 | status | data | args | path | method 103 | --------+---------+------+-------------------+-------- 104 | 200 | payload | bar | /anything?foo=bar | PUT 105 | (1 row) 106 | 107 | -- PATCH 108 | SELECT status, 109 | content::json->>'data' AS data, 110 | content::json->'args'->>'foo' AS args, 111 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 112 | content::json->>'method' AS method 113 | FROM http_patch(current_setting('http.server_host') || '/anything?foo=bar','{"this":"that"}','application/json'); 114 | status | data | args | path | method 115 | --------+-----------------+------+-------------------+-------- 116 | 200 | {"this":"that"} | bar | /anything?foo=bar | PATCH 117 | (1 row) 118 | 119 | -- POST 120 | SELECT status, 121 | content::json->>'data' AS data, 122 | content::json->'args'->>'foo' AS args, 123 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 124 | content::json->>'method' AS method 125 | FROM http_post(current_setting('http.server_host') || '/anything?foo=bar','payload','text/plain'); 126 | status | data | args | path | method 127 | --------+---------+------+-------------------+-------- 128 | 200 | payload | bar | /anything?foo=bar | POST 129 | (1 row) 130 | 131 | -- POST with json data 132 | SELECT status, 133 | content::json->'form'->>'this' AS args, 134 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 135 | content::json->>'method' AS method 136 | FROM http_post(current_setting('http.server_host') || '/anything', jsonb_build_object('this', 'that')); 137 | status | args | path | method 138 | --------+------+-----------+-------- 139 | 200 | that | /anything | POST 140 | (1 row) 141 | 142 | -- POST with data 143 | SELECT status, 144 | content::json->'form'->>'key1' AS key1, 145 | content::json->'form'->>'key2' AS key2, 146 | replace(content::json->>'url', current_setting('http.server_host'),'') AS path, 147 | content::json->>'method' AS method 148 | FROM http_post(current_setting('http.server_host') || '/anything', 'key1=value1&key2=value2','application/x-www-form-urlencoded'); 149 | status | key1 | key2 | path | method 150 | --------+--------+--------+-----------+-------- 151 | 200 | value1 | value2 | /anything | POST 152 | (1 row) 153 | 154 | -- HEAD 155 | SELECT lower(field) AS field, value 156 | FROM ( 157 | SELECT (unnest(headers)).* 158 | FROM http_head(current_setting('http.server_host') || '/response-headers?Abcde=abcde') 159 | ) a 160 | WHERE field ILIKE 'Abcde'; 161 | field | value 162 | -------+------- 163 | abcde | abcde 164 | (1 row) 165 | 166 | -- Follow redirect 167 | SELECT status, 168 | replace((content::json)->>'url', current_setting('http.server_host'),'') AS path 169 | FROM http_get(current_setting('http.server_host') || '/redirect-to?url=get'); 170 | status | path 171 | --------+------ 172 | 200 | /get 173 | (1 row) 174 | 175 | -- Request image 176 | WITH 177 | http AS ( 178 | SELECT * FROM http_get(current_setting('http.server_host') || '/image/png') 179 | ), 180 | headers AS ( 181 | SELECT (unnest(headers)).* FROM http 182 | ) 183 | SELECT 184 | http.content_type, 185 | length(text_to_bytea(http.content)) AS length_binary 186 | FROM http, headers 187 | WHERE field ilike 'Content-Type'; 188 | content_type | length_binary 189 | --------------+--------------- 190 | image/png | 8090 191 | (1 row) 192 | 193 | -- Alter options and and reset them and throw errors 194 | SELECT http_set_curlopt('CURLOPT_PROXY', '127.0.0.1'); 195 | http_set_curlopt 196 | ------------------ 197 | t 198 | (1 row) 199 | 200 | -- Error because proxy is not there 201 | DO $$ 202 | BEGIN 203 | SELECT status FROM http_get(current_setting('http.server_host') || '/status/555'); 204 | EXCEPTION 205 | WHEN OTHERS THEN 206 | RAISE WARNING 'Failed to connect'; 207 | END; 208 | $$; 209 | WARNING: Failed to connect 210 | -- Still an error 211 | DO $$ 212 | BEGIN 213 | SELECT status FROM http_get(current_setting('http.server_host') || '/status/555'); 214 | EXCEPTION 215 | WHEN OTHERS THEN 216 | RAISE WARNING 'Failed to connect'; 217 | END; 218 | $$; 219 | WARNING: Failed to connect 220 | -- Reset options 221 | SELECT http_reset_curlopt(); 222 | http_reset_curlopt 223 | -------------------- 224 | t 225 | (1 row) 226 | 227 | -- Now it should work 228 | SELECT status FROM http_get(current_setting('http.server_host') || '/status/555'); 229 | status 230 | -------- 231 | 555 232 | (1 row) 233 | 234 | -- Alter the default timeout and then run a query that is longer than 235 | -- the default (5s), but shorter than the new timeout 236 | SELECT http_set_curlopt('CURLOPT_TIMEOUT_MS', '10000'); 237 | http_set_curlopt 238 | ------------------ 239 | t 240 | (1 row) 241 | 242 | SELECT status FROM http_get(current_setting('http.server_host') || '/delay/7'); 243 | status 244 | -------- 245 | 200 246 | (1 row) 247 | 248 | -- Test new GUC feature 249 | SET http.CURLOPT_TIMEOUT_MS = '10'; 250 | -- should fail 251 | -- Still an error 252 | DO $$ 253 | BEGIN 254 | SELECT status FROM http_get(current_setting('http.server_host') || '/delay/7'); 255 | EXCEPTION 256 | WHEN OTHERS THEN 257 | RAISE WARNING 'Failed to connect'; 258 | END; 259 | $$; 260 | WARNING: Failed to connect 261 | SET http.CURLOPT_TIMEOUT_MS = '10000'; 262 | --should pass 263 | SELECT status FROM http_get(current_setting('http.server_host') || '/delay/7'); 264 | status 265 | -------- 266 | 200 267 | (1 row) 268 | 269 | -- SET to bogus file 270 | SET http.CURLOPT_CAINFO = '/path/to/somebundle.crt'; 271 | -- should fail 272 | DO $$ 273 | BEGIN 274 | SELECT status FROM http_get('https://postgis.net'); 275 | EXCEPTION 276 | WHEN OTHERS THEN 277 | RAISE WARNING 'Invalid cert file'; 278 | END; 279 | $$; 280 | WARNING: Invalid cert file 281 | -- set to ignore cert 282 | SET http.CURLOPT_SSL_VERIFYPEER = '0'; 283 | -- should pass 284 | SELECT status FROM http_get('https://postgis.net'); 285 | status 286 | -------- 287 | 200 288 | (1 row) 289 | 290 | SHOW http.CURLOPT_CAINFO; 291 | http.curlopt_cainfo 292 | ------------------------- 293 | /path/to/somebundle.crt 294 | (1 row) 295 | 296 | -- reset it 297 | RESET http.CURLOPT_CAINFO; 298 | SELECT status FROM http_get(current_setting('http.server_host') || '/delay/7'); 299 | status 300 | -------- 301 | 200 302 | (1 row) 303 | 304 | -- Check that statement interruption works 305 | SET statement_timeout = 200; 306 | CREATE TEMPORARY TABLE timer AS 307 | SELECT now() AS start; 308 | SELECT * 309 | FROM http_get(current_setting('http.server_host') || '/delay/7'); 310 | ERROR: canceling statement due to user request 311 | SELECT round(extract(epoch FROM now() - start) * 10) AS m 312 | FROM timer; 313 | m 314 | --- 315 | 2 316 | (1 row) 317 | 318 | DROP TABLE timer; 319 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PostgreSQL HTTP Client 2 | 3 | [![CI](https://github.com/pramsey/pgsql-http/workflows/CI/badge.svg)](https://github.com/pramsey/pgsql-http/actions) 4 | 5 | ## Motivation 6 | 7 | Wouldn't it be nice to be able to write a trigger that called a web service? Either to get back a result, or to poke that service into refreshing itself against the new state of the database? 8 | 9 | This extension is for that. 10 | 11 | ## Examples 12 | 13 | URL encode a string. 14 | 15 | ```sql 16 | SELECT urlencode('my special string''s & things?'); 17 | ``` 18 | ``` 19 | urlencode 20 | ------------------------------------- 21 | my+special+string%27s+%26+things%3F 22 | (1 row) 23 | ``` 24 | 25 | URL encode a JSON associative array. 26 | 27 | ```sql 28 | SELECT urlencode(jsonb_build_object('name','Colin & James','rate','50%')); 29 | ``` 30 | ``` 31 | urlencode 32 | ------------------------------------- 33 | name=Colin+%26+James&rate=50%25 34 | (1 row) 35 | ``` 36 | 37 | Run a GET request and see the content. 38 | 39 | ```sql 40 | SELECT content 41 | FROM http_get('http://httpbun.com/ip'); 42 | ``` 43 | ``` 44 | content 45 | ----------------------------- 46 | {"origin":"24.69.186.43"} 47 | (1 row) 48 | ``` 49 | 50 | Run a GET request with an Authorization header. 51 | 52 | ```sql 53 | SELECT content::json->'headers'->>'Authorization' 54 | FROM http(( 55 | 'GET', 56 | 'http://httpbun.com/headers', 57 | http_headers('Authorization','Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9'), 58 | NULL, 59 | NULL 60 | )::http_request); 61 | ``` 62 | ``` 63 | content 64 | ---------------------------------------------- 65 | Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9 66 | (1 row) 67 | ``` 68 | 69 | Read the `status` and `content_type` fields out of a `http_response` object. 70 | 71 | ```sql 72 | SELECT status, content_type 73 | FROM http_get('http://httpbun.com/'); 74 | ``` 75 | ``` 76 | status | content_type 77 | --------+-------------------------- 78 | 200 | text/html; charset=utf-8 79 | (1 row) 80 | ``` 81 | 82 | Show all the `http_header` in an `http_response` object. 83 | 84 | ```sql 85 | SELECT (unnest(headers)).* 86 | FROM http_get('http://httpbun.com/'); 87 | ``` 88 | ``` 89 | field | value 90 | ------------------+-------------------------------------------------- 91 | Server | nginx 92 | Date | Wed, 26 Jul 2023 19:52:51 GMT 93 | Content-Type | text/html 94 | Content-Length | 162 95 | Connection | close 96 | Location | https://httpbun.org 97 | server | nginx 98 | date | Wed, 26 Jul 2023 19:52:51 GMT 99 | content-type | text/html 100 | x-powered-by | httpbun/3c0dc05883dd9212ac38b04705037d50b02f2596 101 | content-encoding | gzip 102 | ``` 103 | 104 | Use the PUT command to send a simple text document to a server. 105 | 106 | ```sql 107 | SELECT status, content_type, content::json->>'data' AS data 108 | FROM http_put('http://httpbun.com/put', 'some text', 'text/plain'); 109 | ``` 110 | ``` 111 | status | content_type | data 112 | --------+------------------+----------- 113 | 200 | application/json | some text 114 | ``` 115 | 116 | Use the PATCH command to send a simple JSON document to a server. 117 | 118 | ```sql 119 | SELECT status, content_type, content::json->>'data' AS data 120 | FROM http_patch('http://httpbun.com/patch', '{"this":"that"}', 'application/json'); 121 | ``` 122 | ``` 123 | status | content_type | data 124 | --------+------------------+------------------ 125 | 200 | application/json | '{"this":"that"}' 126 | ``` 127 | 128 | Use the DELETE command to request resource deletion. 129 | 130 | ```sql 131 | SELECT status, content_type, content::json->>'url' AS url 132 | FROM http_delete('http://httpbun.com/delete'); 133 | ``` 134 | ``` 135 | status | content_type | url 136 | --------+------------------+--------------------------- 137 | 200 | application/json | http://httpbun.com/delete 138 | ``` 139 | 140 | As a shortcut to send data to a GET request, pass a JSONB data argument. 141 | 142 | ```sql 143 | SELECT status, content::json->'args' AS args 144 | FROM http_get('http://httpbun.com/get', 145 | jsonb_build_object('myvar','myval','foo','bar')); 146 | ``` 147 | 148 | To POST to a URL using a data payload instead of parameters embedded in the URL, encode the data in a JSONB as a data payload. 149 | 150 | ```sql 151 | SELECT status, content::json->'form' AS form 152 | FROM http_post('http://httpbun.com/post', 153 | jsonb_build_object('myvar','myval','foo','bar')); 154 | ``` 155 | 156 | To access binary content, you must coerce the content from the default `varchar` representation to a `bytea` representation using the `text_to_bytea()` function, or the `textsend()` function. Using the default `varchar::bytea` cast will **not work**, as the cast will stop the first time it hits a zero-valued byte (common in binary data). 157 | 158 | ```sql 159 | WITH 160 | http AS ( 161 | SELECT * FROM http_get('https://httpbingo.org/image/png') 162 | ), 163 | headers AS ( 164 | SELECT (unnest(headers)).* FROM http 165 | ) 166 | SELECT 167 | http.content_type, 168 | length(text_to_bytea(http.content)) AS length_binary 169 | FROM http, headers 170 | WHERE field ilike 'Content-Type'; 171 | ``` 172 | ``` 173 | content_type | length_binary 174 | --------------+--------------- 175 | image/png | 8090 176 | ``` 177 | Similarly, when using POST to send `bytea` binary content to a service, use the `bytea_to_text` function to prepare the content. 178 | 179 | To access only the headers you can do a HEAD-Request. This will not follow redirections. 180 | 181 | ```sql 182 | SELECT 183 | http.status, 184 | headers.value AS location 185 | FROM 186 | http_head('http://google.com') AS http 187 | LEFT OUTER JOIN LATERAL (SELECT value 188 | FROM unnest(http.headers) 189 | WHERE field = 'Location') AS headers 190 | ON true; 191 | ``` 192 | ``` 193 | status | location 194 | --------+------------------------ 195 | 301 | http://www.google.com/ 196 | ``` 197 | 198 | ## Concepts 199 | 200 | Every HTTP call is a made up of an `http_request` and an `http_response`. 201 | 202 | Composite type "public.http_request" 203 | Column | Type | Modifiers 204 | --------------+-------------------+----------- 205 | method | http_method | 206 | uri | character varying | 207 | headers | http_header[] | 208 | content_type | character varying | 209 | content | character varying | 210 | 211 | Composite type "public.http_response" 212 | Column | Type | Modifiers 213 | --------------+-------------------+----------- 214 | status | integer | 215 | content_type | character varying | 216 | headers | http_header[] | 217 | content | character varying | 218 | 219 | The utility functions, `http_get()`, `http_post()`, `http_put()`, `http_delete()` and `http_head()` are just wrappers around a master function, `http(http_request)` that returns `http_response`. 220 | 221 | The `headers` field for requests and response is a PostgreSQL array of type `http_header` which is just a simple tuple. 222 | 223 | Composite type "public.http_header" 224 | Column | Type | Modifiers 225 | --------+-------------------+----------- 226 | field | character varying | 227 | value | character varying | 228 | 229 | As seen in the examples, you can unspool the array of `http_header` tuples into a result set using the PostgreSQL `unnest()` function on the array. From there you select out the particular header you are interested in. 230 | 231 | ## Functions 232 | 233 | * `http_header(field VARCHAR, value VARCHAR)` returns `http_header` 234 | * `http_headers(field VARCHAR, value VARCHAR, ...)` returns `http_header[]` 235 | * `http(request http_request)` returns `http_response` 236 | * `http_get(uri VARCHAR)` returns `http_response` 237 | * `http_get(uri VARCHAR, data JSONB)` returns `http_response` 238 | * `http_post(uri VARCHAR, content VARCHAR, content_type VARCHAR)` returns `http_response` 239 | * `http_post(uri VARCHAR, data JSONB)` returns `http_response` 240 | * `http_put(uri VARCHAR, content VARCHAR, content_type VARCHAR)` returns `http_response` 241 | * `http_patch(uri VARCHAR, content VARCHAR, content_type VARCHAR)` returns `http_response` 242 | * `http_delete(uri VARCHAR, content VARCHAR, content_type VARCHAR))` returns `http_response` 243 | * `http_head(uri VARCHAR)` returns `http_response` 244 | * `http_set_curlopt(curlopt VARCHAR, value varchar)` returns `boolean` 245 | * `http_reset_curlopt()` returns `boolean` 246 | * `http_list_curlopt()` returns `setof(curlopt text, value text)` 247 | * `urlencode(string VARCHAR)` returns `text` 248 | * `urlencode(data JSONB)` returns `text` 249 | 250 | ## CURL Options 251 | 252 | Select [CURL options](https://curl.se/libcurl/c/curl_easy_setopt.html) are available to set using the SQL `SET` command and the appropriate option name. 253 | 254 | * [CURLOPT_CAINFO](https://curl.se/libcurl/c/CURLOPT_CAINFO.html) 255 | * [CURLOPT_CONNECTTIMEOUT](https://curl.se/libcurl/c/CURLOPT_CONNECTTIMEOUT.html) 256 | * [CURLOPT_CONNECTTIMEOUT_MS](https://curl.se/libcurl/c/CURLOPT_CONNECTTIMEOUT_MS.html) 257 | * [CURLOPT_DNS_SERVERS](https://curl.se/libcurl/c/CURLOPT_DNS_SERVERS.html) 258 | * [CURLOPT_PRE_PROXY](https://curl.se/libcurl/c/CURLOPT_PRE_PROXY.html) 259 | * [CURLOPT_PROXY](https://curl.se/libcurl/c/CURLOPT_PROXY.html) 260 | * [CURLOPT_PROXYPASSWORD](https://curl.se/libcurl/c/CURLOPT_PROXYPASSWORD.html) 261 | * [CURLOPT_PROXYPORT](https://curl.se/libcurl/c/CURLOPT_PROXYPORT.html) 262 | * [CURLOPT_PROXYUSERPWD](https://curl.se/libcurl/c/CURLOPT_PROXYUSERPWD.html) 263 | * [CURLOPT_PROXYUSERNAME](https://curl.se/libcurl/c/CURLOPT_PROXYUSERNAME.html) 264 | * [CURLOPT_PROXY_TLSAUTH_USERNAME](https://curl.se/libcurl/c/CURLOPT_PROXY_TLSAUTH_USERNAME.html) 265 | * [CURLOPT_PROXY_TLSAUTH_PASSWORD](https://curl.se/libcurl/c/CURLOPT_PROXY_TLSAUTH_PASSWORD.html) 266 | * [CURLOPT_PROXY_TLSAUTH_TYPE](https://curl.se/libcurl/c/CURLOPT_PROXY_TLSAUTH_TYPE.html) 267 | * [CURLOPT_SSL_VERIFYHOST](https://curl.se/libcurl/c/CURLOPT_SSL_VERIFYHOST.html) 268 | * [CURLOPT_SSL_VERIFYPEER](https://curl.se/libcurl/c/CURLOPT_SSL_VERIFYPEER.html) 269 | * [CURLOPT_SSLCERT](https://curl.se/libcurl/c/CURLOPT_SSLCERT.html) 270 | * [CURLOPT_SSLCERT_BLOB](https://curl.se/libcurl/c/CURLOPT_SSLCERT_BLOB.html) 271 | * [CURLOPT_SSLCERTTYPE](https://curl.se/libcurl/c/CURLOPT_SSLCERTTYPE.html) 272 | * [CURLOPT_SSLKEY](https://curl.se/libcurl/c/CURLOPT_SSLKEY.html) 273 | * [CURLOPT_SSLKEY_BLOB](https://curl.se/libcurl/c/CURLOPT_SSLKEY_BLOB.html) 274 | * [CURLOPT_TCP_KEEPALIVE](https://curl.se/libcurl/c/CURLOPT_TCP_KEEPALIVE.html) 275 | * [CURLOPT_TCP_KEEPIDLE](https://curl.se/libcurl/c/CURLOPT_TCP_KEEPIDLE.html) 276 | * [CURLOPT_TIMEOUT](https://curl.se/libcurl/c/CURLOPT_TIMEOUT.html) 277 | * [CURLOPT_TIMEOUT_MS](https://curl.se/libcurl/c/CURLOPT_TIMEOUT_MS.html) 278 | * [CURLOPT_TLSAUTH_USERNAME](https://curl.se/libcurl/c/CURLOPT_TLSAUTH_USERNAME.html) 279 | * [CURLOPT_TLSAUTH_PASSWORD](https://curl.se/libcurl/c/CURLOPT_TLSAUTH_PASSWORD.html) 280 | * [CURLOPT_TLSAUTH_TYPE](https://curl.se/libcurl/c/CURLOPT_TLSAUTH_TYPE.html) 281 | * [CURLOPT_USERAGENT](https://curl.se/libcurl/c/CURLOPT_USERAGENT.html) 282 | * [CURLOPT_USERPWD](https://curl.se/libcurl/c/CURLOPT_USERPWD.html) 283 | 284 | For example, 285 | 286 | ```sql 287 | -- Set the curlopt_proxyport option 288 | SET http.curlopt_proxyport = '12345'; 289 | 290 | -- View the curlopt_proxyport option 291 | SHOW http.curlopt_proxyport; 292 | 293 | -- View *all* currently set options 294 | SELECT * FROM http_list_curlopt(); 295 | ``` 296 | 297 | Will set the proxy port option for the lifetime of the database connection. You can reset all CURL options to their defaults using the `http_reset_curlopt()` function. 298 | 299 | You can permanently set the CURL options for a database or role, using the `ALTER DATABASE` and `ALTER ROLE` commands. 300 | 301 | ```sql 302 | -- Applies to all roles in the database 303 | ALTER DATABASE mydb SET http.curlopt_tlsauth_password = 'secret'; 304 | 305 | -- Applies to just one role in the database 306 | ALTER ROLE myapp IN mydb SET http.curlopt_tlsauth_password = 'secret'; 307 | ``` 308 | 309 | ## User Agents 310 | 311 | Using this extension as a background automated process without supervision (e.g as a trigger) may have unintended consequences for other servers. It is considered a best practice to share contact information with your requests, so that administrators can reach you in case your HTTP calls get out of control. 312 | 313 | Certain API policies (e.g. [Wikimedia User-Agent policy](https://foundation.wikimedia.org/wiki/Policy:Wikimedia_Foundation_User-Agent_Policy)) may even require sharing specific contact information with each request. Others may disallow (via `robots.txt`) certain agents they don't recognize. 314 | 315 | For such cases you can set the `CURLOPT_USERAGENT` option 316 | 317 | ```sql 318 | SET http.curlopt_useragent = 'PgBot/2.1 (+http://pgbot.com/bot.html) Contact abuse@pgbot.com'; 319 | 320 | SELECT status, content::json->'headers'->>'User-Agent' 321 | FROM http_get('http://httpbun.com/headers'); 322 | ``` 323 | ``` 324 | status | user_agent 325 | --------+----------------------------------------------------------- 326 | 200 | PgBot/2.1 (+http://pgbot.com/bot.html) Contact abuse@pgbot.com 327 | ``` 328 | 329 | ## Keep-Alive & Timeouts 330 | 331 | By default each request uses a fresh connection and assures that the connection is closed when the request is done. This behavior reduces the chance of consuming system resources (sockets) as the extension runs over extended periods of time. 332 | 333 | High-performance applications may wish to enable keep-alive and connection persistence to reduce latency and enhance throughput. The following GUC variable changes the behavior of the http extension to maintain connections as long as possible: 334 | 335 | ```sql 336 | SET http.curlopt_tcp_keepalive = 1; 337 | ``` 338 | 339 | By default a 5 second timeout is set for the completion of a request. If a different timeout is desired the following GUC variable can be used to set it in milliseconds: 340 | 341 | ```sql 342 | SET http.curlopt_timeout_ms = 200; 343 | ``` 344 | 345 | You can also change the timeout for the connection, to avoid waiting for too long when a service is unavailable: 346 | 347 | ```sql 348 | SET http.curlopt_connecttimeout_ms = 100; 349 | ``` 350 | 351 | When a timeout occurs during a request, a SQL error will be raised: 352 | 353 | ```sql 354 | ERROR: Operation timed out after 200 milliseconds with 0 bytes received 355 | ``` 356 | 357 | ## Installation 358 | 359 | ### Debian / Ubuntu apt.postgresql.org 360 | Replace 17 with your version of PostgreSQL 361 | ``` 362 | apt install postgresql-17-http 363 | ``` 364 | 365 | ### Compile from Source 366 | 367 | #### General Unix 368 | 369 | If you have PostgreSQL devel packages and CURL devel packages installed, you should have `pg_config` and `curl-config` on your path, so you should be able to just run `make` (or `gmake`), then `make install`, then in your database `CREATE EXTENSION http`. 370 | 371 | If you already installed a previous version and you just want to upgrade, then `ALTER EXTENSION http UPDATE`. 372 | 373 | #### Debian / Ubuntu / APT 374 | 375 | Refer to https://wiki.postgresql.org/wiki/Apt for pulling packages from apt.postgresql.org repository. 376 | 377 | ```bash 378 | sudo apt install \ 379 | postgresql-server-dev-14 \ 380 | libcurl4-openssl-dev \ 381 | make \ 382 | g++ 383 | 384 | make 385 | sudo make install 386 | ``` 387 | 388 | If there several PostgreSQL installations available, you might need to edit the Makefile before running `make`: 389 | 390 | ``` 391 | #PG_CONFIG = pg_config 392 | PG_CONFIG = /usr/lib/postgresql/14/bin/pg_config 393 | ``` 394 | 395 | ### Windows 396 | 397 | There is a build available at [postgresonline](http://www.postgresonline.com/journal/archives/371-http-extension.html), not maintained by me. 398 | 399 | ### Testing 400 | 401 | The integration tests are run with `make install && make installcheck` and expect to find a running instance of [httpbin](http://httpbin.org) at port 9080. The easiest way to get that is: 402 | 403 | ``` 404 | docker run -p 9080:80 kennethreitz/httpbin 405 | ``` 406 | 407 | ## Why This is a Bad Idea 408 | 409 | - "What happens if the web page takes a long time to return?" Your SQL call will just wait there until it does. Make sure your web service fails fast. Or (dangerous in a different way) run your query within [pg_background](https://github.com/vibhorkum/pg_background) or on a schedule with [pg_cron](https://github.com/citusdata/pg_cron). 410 | - "What if the web page returns junk?" Your SQL call will have to test for junk before doing anything with the payload. 411 | - "What if the web page never returns?" Set a short timeout, or send a cancel to the request, or just wait forever. 412 | - "What if a user queries a page they shouldn't?" Restrict function access, or just don't install a footgun like this extension where users can access it. 413 | 414 | ## To Do 415 | 416 | - The [background worker](https://www.postgresql.org/docs/current/bgworker.html) support could be used to set up an HTTP request queue, so that pgsql-http can register a request and callback and then return immediately. 417 | 418 | -------------------------------------------------------------------------------- /http.c: -------------------------------------------------------------------------------- 1 | /*********************************************************************** 2 | * 3 | * Project: PgSQL HTTP 4 | * Purpose: Main file. 5 | * 6 | *********************************************************************** 7 | * Copyright 2025 Paul Ramsey 8 | * 9 | * Permission is hereby granted, free of charge, to any person obtaining a 10 | * copy of this software and associated documentation files (the 11 | * "Software"), to deal in the Software without restriction, including 12 | * without limitation the rights to use, copy, modify, merge, publish, 13 | * distribute, sublicense, and/or sell copies of the Software, and to 14 | * permit persons to whom the Software is furnished to do so, subject to 15 | * the following conditions: 16 | * 17 | * The above copyright notice and this permission notice shall be included 18 | * in all copies or substantial portions of the Software. 19 | * 20 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 21 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 22 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 23 | * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 24 | * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 25 | * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 26 | * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 27 | * 28 | ***********************************************************************/ 29 | 30 | /* Constants */ 31 | #define HTTP_VERSION "1.7" 32 | #define HTTP_ENCODING "gzip" 33 | #define CURL_MIN_VERSION 0x071400 /* 7.20.0 */ 34 | 35 | /* System */ 36 | #include 37 | #include 38 | #include 39 | #include /* INT_MAX */ 40 | #include /* SIGINT */ 41 | 42 | /* PostgreSQL */ 43 | #include 44 | #include 45 | #include 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | #include 59 | #include 60 | #include 61 | #include 62 | #include 63 | #include 64 | #include 65 | #include 66 | #include 67 | #include 68 | 69 | #if PG_VERSION_NUM >= 90300 70 | # include 71 | #endif 72 | 73 | #if PG_VERSION_NUM >= 100000 74 | # include 75 | #endif 76 | 77 | #if PG_VERSION_NUM >= 120000 78 | # include 79 | #else 80 | # define table_open(rel, lock) heap_open((rel), (lock)) 81 | # define table_close(rel, lock) heap_close((rel), (lock)) 82 | #endif 83 | 84 | #if PG_VERSION_NUM < 110000 85 | #define PG_GETARG_JSONB_P(x) DatumGetJsonb(PG_GETARG_DATUM(x)) 86 | #endif 87 | 88 | /* CURL */ 89 | #include 90 | 91 | /* Set up PgSQL */ 92 | #ifdef PG_MODULE_MAGIC_EXT 93 | PG_MODULE_MAGIC_EXT( 94 | .name = "http", 95 | .version = HTTP_VERSION 96 | ); 97 | #else 98 | PG_MODULE_MAGIC; 99 | #endif 100 | 101 | /* HTTP request methods we support */ 102 | typedef enum { 103 | HTTP_GET, 104 | HTTP_POST, 105 | HTTP_DELETE, 106 | HTTP_PUT, 107 | HTTP_HEAD, 108 | HTTP_PATCH, 109 | HTTP_UNKNOWN 110 | } http_method; 111 | 112 | /* Components (and postitions) of the http_request tuple type */ 113 | typedef enum { 114 | REQ_METHOD = 0, 115 | REQ_URI = 1, 116 | REQ_HEADERS = 2, 117 | REQ_CONTENT_TYPE = 3, 118 | REQ_CONTENT = 4 119 | } http_request_type; 120 | 121 | /* Components (and postitions) of the http_response tuple type */ 122 | typedef enum { 123 | RESP_STATUS = 0, 124 | RESP_CONTENT_TYPE = 1, 125 | RESP_HEADERS = 2, 126 | RESP_CONTENT = 3 127 | } http_response_type; 128 | 129 | /* Components (and postitions) of the http_header tuple type */ 130 | typedef enum { 131 | HEADER_FIELD = 0, 132 | HEADER_VALUE = 1 133 | } http_header_type; 134 | 135 | /* 136 | * String/Long for strings and numbers, blob only for 137 | * CURLOPT_SSLKEY_BLOB and CURLOPT_SSLCERT_BLOB 138 | */ 139 | typedef enum { 140 | CURLOPT_STRING, 141 | CURLOPT_LONG, 142 | CURLOPT_BLOB 143 | } http_curlopt_type; 144 | 145 | /* CURLOPT string/enum value mapping */ 146 | typedef struct { 147 | CURLoption curlopt; 148 | http_curlopt_type curlopt_type; 149 | bool superuser_only; 150 | char *curlopt_str; 151 | char *curlopt_val; 152 | char *curlopt_guc; 153 | } http_curlopt; 154 | 155 | 156 | /* CURLOPT values we allow user to set at run-time */ 157 | /* Be careful adding these, as they can be a security risk */ 158 | static http_curlopt settable_curlopts[] = { 159 | { CURLOPT_CAINFO, CURLOPT_STRING, false, "CURLOPT_CAINFO", NULL, NULL }, 160 | { CURLOPT_TIMEOUT, CURLOPT_LONG, false, "CURLOPT_TIMEOUT", NULL, NULL }, 161 | { CURLOPT_TIMEOUT_MS, CURLOPT_LONG, false, "CURLOPT_TIMEOUT_MS", NULL, NULL }, 162 | { CURLOPT_CONNECTTIMEOUT, CURLOPT_LONG, false, "CURLOPT_CONNECTTIMEOUT", NULL, NULL }, 163 | { CURLOPT_CONNECTTIMEOUT_MS, CURLOPT_LONG, false, "CURLOPT_CONNECTTIMEOUT_MS", NULL, NULL }, 164 | { CURLOPT_USERAGENT, CURLOPT_STRING, false, "CURLOPT_USERAGENT", NULL, NULL }, 165 | { CURLOPT_USERPWD, CURLOPT_STRING, false, "CURLOPT_USERPWD", NULL, NULL }, 166 | { CURLOPT_IPRESOLVE, CURLOPT_LONG, false, "CURLOPT_IPRESOLVE", NULL, NULL }, 167 | #if LIBCURL_VERSION_NUM >= 0x070903 /* 7.9.3 */ 168 | { CURLOPT_SSLCERTTYPE, CURLOPT_STRING, false, "CURLOPT_SSLCERTTYPE", NULL, NULL }, 169 | #endif 170 | #if LIBCURL_VERSION_NUM >= 0x070e01 /* 7.14.1 */ 171 | { CURLOPT_PROXY, CURLOPT_STRING, false, "CURLOPT_PROXY", NULL, NULL }, 172 | { CURLOPT_PROXYPORT, CURLOPT_LONG, false, "CURLOPT_PROXYPORT", NULL, NULL }, 173 | #endif 174 | #if LIBCURL_VERSION_NUM >= 0x071301 /* 7.19.1 */ 175 | { CURLOPT_PROXYUSERNAME, CURLOPT_STRING, false, "CURLOPT_PROXYUSERNAME", NULL, NULL }, 176 | { CURLOPT_PROXYPASSWORD, CURLOPT_STRING, false, "CURLOPT_PROXYPASSWORD", NULL, NULL }, 177 | #endif 178 | #if LIBCURL_VERSION_NUM >= 0x071504 /* 7.21.4 */ 179 | { CURLOPT_TLSAUTH_USERNAME, CURLOPT_STRING, false, "CURLOPT_TLSAUTH_USERNAME", NULL, NULL }, 180 | { CURLOPT_TLSAUTH_PASSWORD, CURLOPT_STRING, false, "CURLOPT_TLSAUTH_PASSWORD", NULL, NULL }, 181 | { CURLOPT_TLSAUTH_TYPE, CURLOPT_STRING, false, "CURLOPT_TLSAUTH_TYPE", NULL, NULL }, 182 | #endif 183 | #if LIBCURL_VERSION_NUM >= 0x071800 /* 7.24.0 */ 184 | { CURLOPT_DNS_SERVERS, CURLOPT_STRING, false, "CURLOPT_DNS_SERVERS", NULL, NULL }, 185 | #endif 186 | #if LIBCURL_VERSION_NUM >= 0x071900 /* 7.25.0 */ 187 | { CURLOPT_TCP_KEEPALIVE, CURLOPT_LONG, false, "CURLOPT_TCP_KEEPALIVE", NULL, NULL }, 188 | { CURLOPT_TCP_KEEPIDLE, CURLOPT_LONG, false, "CURLOPT_TCP_KEEPIDLE", NULL, NULL }, 189 | #endif 190 | #if LIBCURL_VERSION_NUM >= 0x072500 /* 7.37.0 */ 191 | { CURLOPT_SSL_VERIFYHOST, CURLOPT_LONG, false, "CURLOPT_SSL_VERIFYHOST", NULL, NULL }, 192 | { CURLOPT_SSL_VERIFYPEER, CURLOPT_LONG, false, "CURLOPT_SSL_VERIFYPEER", NULL, NULL }, 193 | #endif 194 | { CURLOPT_SSLCERT, CURLOPT_STRING, false, "CURLOPT_SSLCERT", NULL, NULL }, 195 | { CURLOPT_SSLKEY, CURLOPT_STRING, false, "CURLOPT_SSLKEY", NULL, NULL }, 196 | #if LIBCURL_VERSION_NUM >= 0x073400 /* 7.52.0 */ 197 | { CURLOPT_PRE_PROXY, CURLOPT_STRING, false, "CURLOPT_PRE_PROXY", NULL, NULL }, 198 | { CURLOPT_PROXY_TLSAUTH_USERNAME, CURLOPT_STRING, false, "CURLOPT_PROXY_CAINFO", NULL, NULL }, 199 | { CURLOPT_PROXY_TLSAUTH_USERNAME, CURLOPT_STRING, false, "CURLOPT_PROXY_TLSAUTH_USERNAME", NULL, NULL }, 200 | { CURLOPT_PROXY_TLSAUTH_PASSWORD, CURLOPT_STRING, false, "CURLOPT_PROXY_TLSAUTH_PASSWORD", NULL, NULL }, 201 | { CURLOPT_PROXY_TLSAUTH_TYPE, CURLOPT_STRING, false, "CURLOPT_PROXY_TLSAUTH_TYPE", NULL, NULL }, 202 | #endif 203 | #if LIBCURL_VERSION_NUM >= 0x074700 /* 7.71.0 */ 204 | { CURLOPT_SSLKEY_BLOB, CURLOPT_BLOB, false, "CURLOPT_SSLKEY_BLOB", NULL, NULL }, 205 | { CURLOPT_SSLCERT_BLOB, CURLOPT_BLOB, false, "CURLOPT_SSLCERT_BLOB", NULL, NULL }, 206 | #endif 207 | { 0, 0, false, NULL, NULL, NULL } /* Array null terminator */ 208 | }; 209 | 210 | 211 | /* Function signatures */ 212 | void _PG_init(void); 213 | void _PG_fini(void); 214 | static size_t http_writeback(void *contents, size_t size, size_t nmemb, void *userp); 215 | static size_t http_readback(void *buffer, size_t size, size_t nitems, void *instream); 216 | 217 | /* Global variables */ 218 | static CURL * g_http_handle = NULL; 219 | 220 | /* 221 | * Interrupt support is dependent on CURLOPT_XFERINFOFUNCTION which 222 | * is only available from 7.39.0 and up 223 | */ 224 | #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ 225 | 226 | /* 227 | * To support request interruption, we have libcurl run the progress meter 228 | * callback frequently, and here we watch to see if PgSQL has flipped 229 | * the global QueryCancelPending || ProcDiePending flags. 230 | * Curl should then return CURLE_ABORTED_BY_CALLBACK 231 | * to the curl_easy_perform() call. 232 | */ 233 | static int 234 | http_progress_callback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow) 235 | { 236 | #ifdef WIN32 237 | if (UNBLOCKED_SIGNAL_QUEUE()) 238 | pgwin32_dispatch_queued_signals(); 239 | #endif 240 | /* Check the PgSQL global flags */ 241 | return QueryCancelPending || ProcDiePending; 242 | } 243 | 244 | #endif /* 7.39.0 */ 245 | 246 | #undef HTTP_MEM_CALLBACKS 247 | #ifdef HTTP_MEM_CALLBACKS 248 | static void * 249 | http_calloc(size_t a, size_t b) 250 | { 251 | if (a>0 && b>0) 252 | return palloc0(a*b); 253 | else 254 | return NULL; 255 | } 256 | 257 | static void 258 | http_free(void *a) 259 | { 260 | if (a) 261 | pfree(a); 262 | } 263 | 264 | static void * 265 | http_realloc(void *a, size_t sz) 266 | { 267 | if (a && sz) 268 | return repalloc(a, sz); 269 | else if (sz) 270 | return palloc(sz); 271 | else 272 | return a; 273 | } 274 | 275 | static void * 276 | http_malloc(size_t sz) 277 | { 278 | return sz ? palloc(sz) : NULL; 279 | } 280 | #endif 281 | 282 | 283 | static char * 284 | http_strtolower(const char *input) 285 | { 286 | char *ptr, *output; 287 | if (input == NULL) 288 | return NULL; 289 | 290 | /* Allocate memory for the output string */ 291 | output = palloc(strlen(input) + 1); 292 | ptr = output; 293 | 294 | while (*input) 295 | { 296 | *ptr++ = tolower((unsigned char) *input); 297 | input++; 298 | } 299 | 300 | *ptr = '\0'; // Null-terminate the string 301 | return output; 302 | } 303 | 304 | 305 | #if PG_VERSION_NUM < 160000 306 | static void * 307 | guc_malloc(int elevel, size_t size) 308 | { 309 | void *data; 310 | 311 | /* Avoid unportable behavior of malloc(0) */ 312 | if (size == 0) 313 | size = 1; 314 | data = malloc(size); 315 | if (data == NULL) 316 | ereport(elevel, 317 | (errcode(ERRCODE_OUT_OF_MEMORY), 318 | errmsg("out of memory"))); 319 | return data; 320 | } 321 | 322 | static char * 323 | guc_strdup(int elevel, const char *src) 324 | { 325 | size_t len = strlen(src) + 1; 326 | char *dup = guc_malloc(elevel, len); 327 | memcpy(dup, src, len); 328 | return dup; 329 | } 330 | 331 | static void 332 | guc_free(void *ptr) 333 | { 334 | free(ptr); 335 | } 336 | #endif 337 | 338 | 339 | static void 340 | http_guc_init_opt(http_curlopt *opt) 341 | { 342 | char *opt_name_lower = http_strtolower(opt->curlopt_str); 343 | char *opt_name = psprintf("http.%s", opt_name_lower); 344 | 345 | const char *url_tmpl = "https://curl.se/libcurl/c/%s.html"; 346 | char *opt_url = psprintf(url_tmpl, opt->curlopt_str); 347 | opt->curlopt_guc = guc_strdup(ERROR, opt_name); 348 | 349 | DefineCustomStringVariable( 350 | opt_name, // const char *name 351 | guc_strdup(ERROR, opt_url), // const char *short_desc 352 | NULL, // const char *long_desc 353 | &(opt->curlopt_val), // char **valueAddr 354 | NULL, // const char *bootValue 355 | opt->superuser_only ? PGC_SUSET : PGC_USERSET, // GucContext context 356 | 0, // int flags 357 | NULL, // GucStringCheckHook check_hook 358 | NULL, // GucStringAssignHook assign_hook 359 | NULL // GucShowHook show_hook 360 | ); 361 | 362 | pfree(opt_name_lower); 363 | pfree(opt_name); 364 | pfree(opt_url); 365 | 366 | /* 367 | * Backwards compatibility, retain the old GUC 368 | * http.keepalive name for now. 369 | */ 370 | if (opt->curlopt == CURLOPT_TCP_KEEPALIVE) 371 | { 372 | DefineCustomStringVariable( 373 | "http.keepalive", 374 | guc_strdup(ERROR, "https://curl.se/libcurl/c/CURLOPT_TCP_KEEPALIVE.html"), 375 | NULL, 376 | &(opt->curlopt_val), 377 | NULL, 378 | opt->superuser_only ? PGC_SUSET : PGC_USERSET, 379 | 0, NULL, NULL, NULL 380 | ); 381 | } 382 | 383 | /* 384 | * Backwards compatibility, retain the old GUC 385 | * http.timeout_msec name for now. 386 | */ 387 | if (opt->curlopt == CURLOPT_TIMEOUT_MS) 388 | { 389 | DefineCustomStringVariable( 390 | "http.timeout_msec", 391 | guc_strdup(ERROR, "https://curl.se/libcurl/c/CURLOPT_TIMEOUT_MS.html"), 392 | NULL, 393 | &(opt->curlopt_val), 394 | NULL, 395 | opt->superuser_only ? PGC_SUSET : PGC_USERSET, 396 | 0, NULL, NULL, NULL 397 | ); 398 | } 399 | } 400 | 401 | static void 402 | http_guc_init() 403 | { 404 | http_curlopt *opt = settable_curlopts; 405 | while (opt->curlopt) 406 | { 407 | http_guc_init_opt(opt); 408 | opt++; 409 | } 410 | } 411 | 412 | 413 | /* Startup */ 414 | void _PG_init(void) 415 | { 416 | 417 | /* 418 | * Initialize the DefineCustomStringVariable GUC 419 | * functions to allow "SET http.curlopt_var = value" 420 | * to manipulate CURL options. 421 | */ 422 | http_guc_init(); 423 | 424 | #ifdef HTTP_MEM_CALLBACKS 425 | /* 426 | * Use PgSQL memory management in Curl 427 | * Warning, https://curl.se/libcurl/c/curl_global_init_mem.html 428 | * notes "If you are using libcurl from multiple threads or libcurl 429 | * was built with the threaded resolver option then the callback 430 | * functions must be thread safe." PgSQL isn't multi-threaded, 431 | * but we have no control over whether the "threaded resolver" is 432 | * in use. We may need a semaphor to ensure our callbacks are 433 | * accessed sequentially only. 434 | */ 435 | curl_global_init_mem(CURL_GLOBAL_ALL, http_malloc, http_free, http_realloc, pstrdup, http_calloc); 436 | #else 437 | /* Set up Curl! */ 438 | curl_global_init(CURL_GLOBAL_ALL); 439 | #endif 440 | 441 | 442 | } 443 | 444 | /* Tear-down */ 445 | void _PG_fini(void) 446 | { 447 | if (g_http_handle) 448 | { 449 | curl_easy_cleanup(g_http_handle); 450 | g_http_handle = NULL; 451 | } 452 | 453 | curl_global_cleanup(); 454 | elog(NOTICE, "Goodbye from HTTP %s", HTTP_VERSION); 455 | } 456 | 457 | /** 458 | * This function is passed into CURL as the CURLOPT_WRITEFUNCTION, 459 | * this allows the return values to be held in memory, in our case in a string. 460 | */ 461 | static size_t 462 | http_writeback(void *contents, size_t size, size_t nmemb, void *userp) 463 | { 464 | size_t realsize = size * nmemb; 465 | StringInfo si = (StringInfo)userp; 466 | appendBinaryStringInfo(si, (const char*)contents, (int)realsize); 467 | return realsize; 468 | } 469 | 470 | /** 471 | * This function is passed into CURL as the CURLOPT_READFUNCTION, 472 | * this allows the PUT operation to read the data it needs. We 473 | * pass a StringInfo as our input, and per the callback contract 474 | * return the number of bytes read at each call. 475 | */ 476 | static size_t 477 | http_readback(void *buffer, size_t size, size_t nitems, void *instream) 478 | { 479 | size_t reqsize = size * nitems; 480 | StringInfo si = (StringInfo)instream; 481 | size_t remaining = si->len - si->cursor; 482 | size_t readsize = Min(reqsize, remaining); 483 | memcpy(buffer, si->data + si->cursor, readsize); 484 | si->cursor += readsize; 485 | return readsize; 486 | } 487 | 488 | static void 489 | http_error(CURLcode err, const char *error_buffer) 490 | { 491 | if ( strlen(error_buffer) > 0 ) 492 | ereport(ERROR, (errmsg("%s", error_buffer))); 493 | else 494 | ereport(ERROR, (errmsg("%s", curl_easy_strerror(err)))); 495 | } 496 | 497 | /* Utility macro to try a setopt and catch an error */ 498 | #define CURL_SETOPT(handle, opt, value) do { \ 499 | err = curl_easy_setopt((handle), (opt), (value)); \ 500 | if ( err != CURLE_OK ) \ 501 | { \ 502 | http_error(err, http_error_buffer); \ 503 | PG_RETURN_NULL(); \ 504 | } \ 505 | } while (0); 506 | 507 | 508 | /** 509 | * Convert a request type string into the appropriate enumeration value. 510 | */ 511 | static http_method 512 | request_type(const char *method) 513 | { 514 | if ( strcasecmp(method, "GET") == 0 ) 515 | return HTTP_GET; 516 | else if ( strcasecmp(method, "POST") == 0 ) 517 | return HTTP_POST; 518 | else if ( strcasecmp(method, "PUT") == 0 ) 519 | return HTTP_PUT; 520 | else if ( strcasecmp(method, "DELETE") == 0 ) 521 | return HTTP_DELETE; 522 | else if ( strcasecmp(method, "HEAD") == 0 ) 523 | return HTTP_HEAD; 524 | else if ( strcasecmp(method, "PATCH") == 0 ) 525 | return HTTP_PATCH; 526 | else 527 | return HTTP_UNKNOWN; 528 | } 529 | 530 | /** 531 | * Given a field name and value, output a http_header tuple. 532 | */ 533 | static Datum 534 | header_tuple(TupleDesc header_tuple_desc, const char *field, const char *value) 535 | { 536 | HeapTuple header_tuple; 537 | int ncolumns; 538 | Datum *header_values; 539 | bool *header_nulls; 540 | 541 | /* Prepare our return object */ 542 | ncolumns = header_tuple_desc->natts; 543 | header_values = palloc0(sizeof(Datum)*ncolumns); 544 | header_nulls = palloc0(sizeof(bool)*ncolumns); 545 | 546 | header_values[HEADER_FIELD] = CStringGetTextDatum(field); 547 | header_nulls[HEADER_FIELD] = false; 548 | header_values[HEADER_VALUE] = CStringGetTextDatum(value); 549 | header_nulls[HEADER_VALUE] = false; 550 | 551 | /* Build up a tuple from values/nulls lists */ 552 | header_tuple = heap_form_tuple(header_tuple_desc, header_values, header_nulls); 553 | return HeapTupleGetDatum(header_tuple); 554 | } 555 | 556 | /** 557 | * Our own implementation of strcasestr. 558 | */ 559 | static char * 560 | http_strcasestr(const char *s, const char *find) 561 | { 562 | char c, sc; 563 | size_t len; 564 | 565 | if ((c = *find++) != 0) 566 | { 567 | c = tolower((unsigned char)c); 568 | len = strlen(find); 569 | do 570 | { 571 | do 572 | { 573 | if ((sc = *s++) == 0) 574 | return (NULL); 575 | } 576 | while ((char)tolower((unsigned char)sc) != c); 577 | } 578 | while (strncasecmp(s, find, len) != 0); 579 | s--; 580 | } 581 | return ((char *)s); 582 | } 583 | 584 | /** 585 | * Quick and dirty, remove all \r from a StringInfo. 586 | */ 587 | static void 588 | string_info_remove_cr(StringInfo si) 589 | { 590 | int i = 0, j = 0; 591 | while ( si->data[i] ) 592 | { 593 | if ( si->data[i] != '\r' ) 594 | si->data[j++] = si->data[i++]; 595 | else 596 | i++; 597 | } 598 | si->data[j] = '\0'; 599 | si->len -= i-j; 600 | return; 601 | } 602 | 603 | /** 604 | * Add an array of http_header tuples into a Curl string list. 605 | */ 606 | static struct curl_slist * 607 | header_array_to_slist(ArrayType *array, struct curl_slist *headers) 608 | { 609 | ArrayIterator iterator; 610 | Datum value; 611 | bool isnull; 612 | 613 | #if PG_VERSION_NUM >= 90500 614 | iterator = array_create_iterator(array, 0, NULL); 615 | #else 616 | iterator = array_create_iterator(array, 0); 617 | #endif 618 | 619 | while (array_iterate(iterator, &value, &isnull)) 620 | { 621 | HeapTupleHeader rec; 622 | HeapTupleData tuple; 623 | Oid tup_type; 624 | int32 tup_typmod, ncolumns; 625 | TupleDesc tup_desc; 626 | size_t tup_len; 627 | Datum *values; 628 | bool *nulls; 629 | 630 | /* Skip null array items */ 631 | if ( isnull ) 632 | continue; 633 | 634 | rec = DatumGetHeapTupleHeader(value); 635 | tup_type = HeapTupleHeaderGetTypeId(rec); 636 | tup_typmod = HeapTupleHeaderGetTypMod(rec); 637 | tup_len = HeapTupleHeaderGetDatumLength(rec); 638 | tup_desc = lookup_rowtype_tupdesc(tup_type, tup_typmod); 639 | ncolumns = tup_desc->natts; 640 | 641 | /* Prepare for values / nulls to hold the data */ 642 | values = (Datum *) palloc0(ncolumns * sizeof(Datum)); 643 | nulls = (bool *) palloc0(ncolumns * sizeof(bool)); 644 | 645 | /* Build a temporary HeapTuple control structure */ 646 | tuple.t_len = tup_len; 647 | ItemPointerSetInvalid(&(tuple.t_self)); 648 | tuple.t_tableOid = InvalidOid; 649 | tuple.t_data = rec; 650 | 651 | /* Break down the tuple into values/nulls lists */ 652 | heap_deform_tuple(&tuple, tup_desc, values, nulls); 653 | 654 | /* Convert the data into a header */ 655 | /* TODO: Ensure the header list is unique? Or leave that to the */ 656 | /* server to deal with. */ 657 | if ( ! nulls[HEADER_FIELD] ) 658 | { 659 | size_t total_len = 0; 660 | char *buffer = NULL; 661 | char *header_val; 662 | char *header_fld = TextDatumGetCString(values[HEADER_FIELD]); 663 | 664 | /* Don't process "content-type" in the optional headers */ 665 | if ( strlen(header_fld) <= 0 || strncasecmp(header_fld, "Content-Type", 12) == 0 ) 666 | { 667 | elog(NOTICE, "'Content-Type' is not supported as an optional header"); 668 | continue; 669 | } 670 | 671 | if ( nulls[HEADER_VALUE] ) 672 | header_val = pstrdup(""); 673 | else 674 | header_val = TextDatumGetCString(values[HEADER_VALUE]); 675 | total_len = strlen(header_val) + strlen(header_fld) + sizeof(char) + sizeof(": "); 676 | buffer = palloc(total_len); 677 | if (buffer) 678 | { 679 | snprintf(buffer, total_len, "%s: %s", header_fld, header_val); 680 | elog(DEBUG2, "pgsql-http: optional request header '%s'", buffer); 681 | headers = curl_slist_append(headers, buffer); 682 | pfree(buffer); 683 | } 684 | else 685 | { 686 | elog(ERROR, "pgsql-http: palloc(%zu) failure", total_len); 687 | } 688 | pfree(header_fld); 689 | pfree(header_val); 690 | } 691 | 692 | /* Free all the temporary structures */ 693 | ReleaseTupleDesc(tup_desc); 694 | pfree(values); 695 | pfree(nulls); 696 | } 697 | array_free_iterator(iterator); 698 | 699 | return headers; 700 | } 701 | /** 702 | * This function is now exposed in PG16 and above 703 | * so no need to redefine it for PG16 and above 704 | */ 705 | #if PG_VERSION_NUM < 160000 706 | /** 707 | * Look up the namespace the extension is installed in 708 | */ 709 | static Oid 710 | get_extension_schema(Oid ext_oid) 711 | { 712 | Oid result; 713 | SysScanDesc scandesc; 714 | HeapTuple tuple; 715 | ScanKeyData entry[1]; 716 | #if PG_VERSION_NUM >= 120000 717 | Oid pg_extension_oid = Anum_pg_extension_oid; 718 | #else 719 | Oid pg_extension_oid = ObjectIdAttributeNumber; 720 | #endif 721 | Relation rel = table_open(ExtensionRelationId, AccessShareLock); 722 | 723 | ScanKeyInit(&entry[0], 724 | pg_extension_oid, 725 | BTEqualStrategyNumber, F_OIDEQ, 726 | ObjectIdGetDatum(ext_oid)); 727 | 728 | scandesc = systable_beginscan(rel, ExtensionOidIndexId, true, 729 | NULL, 1, entry); 730 | 731 | tuple = systable_getnext(scandesc); 732 | 733 | /* We assume that there can be at most one matching tuple */ 734 | if (HeapTupleIsValid(tuple)) 735 | result = ((Form_pg_extension) GETSTRUCT(tuple))->extnamespace; 736 | else 737 | result = InvalidOid; 738 | 739 | systable_endscan(scandesc); 740 | 741 | table_close(rel, AccessShareLock); 742 | 743 | return result; 744 | } 745 | #endif 746 | 747 | /** 748 | * Look up the tuple description for a extension-defined type, 749 | * avoiding the pitfalls of using relations that are not part 750 | * of the extension, but share the same name as the relation 751 | * of interest. 752 | */ 753 | static TupleDesc 754 | typname_get_tupledesc(const char *extname, const char *typname) 755 | { 756 | Oid extoid = get_extension_oid(extname, true); 757 | Oid extschemaoid; 758 | Oid typoid; 759 | 760 | if ( ! OidIsValid(extoid) ) 761 | elog(ERROR, "could not lookup '%s' extension oid", extname); 762 | 763 | extschemaoid = get_extension_schema(extoid); 764 | 765 | #if PG_VERSION_NUM >= 120000 766 | typoid = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, 767 | PointerGetDatum(typname), 768 | ObjectIdGetDatum(extschemaoid)); 769 | #else 770 | typoid = GetSysCacheOid2(TYPENAMENSP, 771 | PointerGetDatum(typname), 772 | ObjectIdGetDatum(extschemaoid)); 773 | #endif 774 | 775 | if ( OidIsValid(typoid) ) 776 | { 777 | // Oid typ_oid = get_typ_typrelid(rel_oid); 778 | Oid relextoid = getExtensionOfObject(TypeRelationId, typoid); 779 | if ( relextoid == extoid ) 780 | { 781 | return TypeGetTupleDesc(typoid, NIL); 782 | } 783 | } 784 | 785 | elog(ERROR, "could not lookup '%s' tuple desc", typname); 786 | } 787 | 788 | 789 | #define RVSZ 8192 /* Max length of header element */ 790 | 791 | /** 792 | * Convert a string of headers separated by newlines/CRs into an 793 | * array of http_header tuples. 794 | */ 795 | static ArrayType * 796 | header_string_to_array(StringInfo si) 797 | { 798 | /* Array building */ 799 | size_t arr_nelems = 0; 800 | size_t arr_elems_size = 8; 801 | Datum *arr_elems = palloc0(arr_elems_size*sizeof(Datum)); 802 | Oid elem_type; 803 | int16 elem_len; 804 | bool elem_byval; 805 | char elem_align; 806 | 807 | /* Header handling */ 808 | TupleDesc header_tuple_desc = NULL; 809 | 810 | /* Regex support */ 811 | const char *regex_pattern = "^([^ \t\r\n\v\f]+): ?([^ \t\r\n\v\f]+.*)$"; 812 | regex_t regex; 813 | regmatch_t pmatch[3]; 814 | int reti; 815 | char rv1[RVSZ]; 816 | char rv2[RVSZ]; 817 | 818 | /* Compile the regular expression */ 819 | reti = regcomp(®ex, regex_pattern, REG_ICASE | REG_EXTENDED | REG_NEWLINE ); 820 | if ( reti ) 821 | elog(ERROR, "Unable to compile regex pattern '%s'", regex_pattern); 822 | 823 | /* Lookup the tuple defn */ 824 | header_tuple_desc = typname_get_tupledesc("http", "http_header"); 825 | 826 | /* Prepare array building metadata */ 827 | elem_type = header_tuple_desc->tdtypeid; 828 | get_typlenbyvalalign(elem_type, &elem_len, &elem_byval, &elem_align); 829 | 830 | /* Loop through string, matching regex pattern */ 831 | si->cursor = 0; 832 | while ( ! regexec(®ex, si->data+si->cursor, 3, pmatch, 0) ) 833 | { 834 | /* Read the regex match results */ 835 | int eo0 = pmatch[0].rm_eo; 836 | int so1 = pmatch[1].rm_so; 837 | int eo1 = pmatch[1].rm_eo; 838 | int so2 = pmatch[2].rm_so; 839 | int eo2 = pmatch[2].rm_eo; 840 | 841 | /* Copy the matched portions out of the string */ 842 | memcpy(rv1, si->data+si->cursor+so1, Min(eo1-so1, RVSZ)); 843 | rv1[eo1-so1] = '\0'; 844 | memcpy(rv2, si->data+si->cursor+so2, Min(eo2-so2, RVSZ)); 845 | rv2[eo2-so2] = '\0'; 846 | 847 | /* Move forward for next match */ 848 | si->cursor += eo0; 849 | 850 | /* Increase elements array size if necessary */ 851 | if ( arr_nelems >= arr_elems_size ) 852 | { 853 | arr_elems_size *= 2; 854 | arr_elems = repalloc(arr_elems, arr_elems_size*sizeof(Datum)); 855 | } 856 | arr_elems[arr_nelems] = header_tuple(header_tuple_desc, rv1, rv2); 857 | arr_nelems++; 858 | } 859 | 860 | regfree(®ex); 861 | ReleaseTupleDesc(header_tuple_desc); 862 | return construct_array(arr_elems, arr_nelems, elem_type, elem_len, elem_byval, elem_align); 863 | } 864 | 865 | /* Check/log version info */ 866 | static void 867 | http_check_curl_version(const curl_version_info_data *version_info) 868 | { 869 | elog(DEBUG2, "pgsql-http: curl version %s", version_info->version); 870 | elog(DEBUG2, "pgsql-http: curl version number 0x%x", version_info->version_num); 871 | elog(DEBUG2, "pgsql-http: ssl version %s", version_info->ssl_version); 872 | 873 | if ( version_info->version_num < CURL_MIN_VERSION ) 874 | { 875 | elog(ERROR, "pgsql-http requires Curl version 0.7.20 or higher"); 876 | } 877 | } 878 | 879 | 880 | static bool 881 | curlopt_is_set(CURLoption curlopt) 882 | { 883 | http_curlopt *opt = settable_curlopts; 884 | while (opt->curlopt) 885 | { 886 | if (opt->curlopt == curlopt && opt->curlopt_val) 887 | return true; 888 | opt++; 889 | } 890 | return false; 891 | } 892 | 893 | 894 | static bool 895 | set_curlopt(CURL* handle, const http_curlopt *opt) 896 | { 897 | CURLcode err = CURLE_OK; 898 | char http_error_buffer[CURL_ERROR_SIZE] = "\0"; 899 | 900 | memset(http_error_buffer, 0, sizeof(http_error_buffer)); 901 | 902 | /* Argument is a string */ 903 | if (opt->curlopt_type == CURLOPT_STRING) 904 | { 905 | err = curl_easy_setopt(handle, opt->curlopt, opt->curlopt_val); 906 | elog(DEBUG2, "pgsql-http: set '%s' to value '%s', return value = %d", opt->curlopt_guc, opt->curlopt_val, err); 907 | } 908 | /* Argument is a long */ 909 | else if (opt->curlopt_type == CURLOPT_LONG) 910 | { 911 | long value_long; 912 | errno = 0; 913 | value_long = strtol(opt->curlopt_val, NULL, 10); 914 | if ( errno == EINVAL || errno == ERANGE ) 915 | elog(ERROR, "invalid integer provided for '%s'", opt->curlopt_guc); 916 | 917 | err = curl_easy_setopt(handle, opt->curlopt, value_long); 918 | elog(DEBUG2, "pgsql-http: set '%s' to value '%ld', return value = %d", opt->curlopt_guc, value_long, err); 919 | } 920 | #if LIBCURL_VERSION_NUM >= 0x074700 /* 7.71.0 */ 921 | /* Only used for CURLOPT_SSLKEY_BLOB and CURLOPT_SSLCERT_BLOB */ 922 | else if (opt->curlopt_type == CURLOPT_BLOB) 923 | { 924 | struct curl_blob blob; 925 | blob.len = strlen(opt->curlopt_val) + 1; 926 | blob.data = opt->curlopt_val; 927 | blob.flags = CURL_BLOB_COPY; 928 | 929 | err = curl_easy_setopt(handle, CURLOPT_SSLKEYTYPE, "PEM"); 930 | elog(DEBUG2, "pgsql-http: set 'CURLOPT_SSLKEYTYPE' to value 'PEM', return value = %d", err); 931 | 932 | err = curl_easy_setopt(handle, opt->curlopt, &blob); 933 | elog(DEBUG2, "pgsql-http: set '%s' to value '%s', return value = %d", opt->curlopt_guc, opt->curlopt_val, err); 934 | } 935 | #endif 936 | else 937 | { 938 | /* Never get here */ 939 | elog(ERROR, "invalid curlopt_type, '%d'", opt->curlopt_type); 940 | } 941 | 942 | if ( err != CURLE_OK ) 943 | { 944 | http_error(err, http_error_buffer); 945 | return false; 946 | } 947 | return true; 948 | } 949 | 950 | /* Check/create the global CURL* handle */ 951 | static CURL * 952 | http_get_handle() 953 | { 954 | CURL *handle = g_http_handle; 955 | http_curlopt *opt = settable_curlopts; 956 | 957 | /* Initialize the global handle if needed */ 958 | if (!handle) 959 | { 960 | handle = curl_easy_init(); 961 | } 962 | /* Always reset because we are going to fill in the user */ 963 | /* set options down below */ 964 | else 965 | { 966 | curl_easy_reset(handle); 967 | } 968 | 969 | /* Always want a default fast (1 second) connection timeout */ 970 | /* User can over-ride with http_set_curlopt() if they wish */ 971 | curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, 1000); 972 | curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, 5000); 973 | 974 | /* Set the user agent. If not set, use PG_VERSION as default */ 975 | curl_easy_setopt(handle, CURLOPT_USERAGENT, PG_VERSION_STR); 976 | 977 | if (!handle) 978 | ereport(ERROR, (errmsg("Unable to initialize CURL"))); 979 | 980 | /* Bring in any options the user has set this session */ 981 | while (opt->curlopt) 982 | { 983 | /* Option value is already set */ 984 | if (opt->curlopt_val) 985 | set_curlopt(handle, opt); 986 | opt++; 987 | } 988 | 989 | g_http_handle = handle; 990 | return handle; 991 | } 992 | 993 | 994 | /** 995 | * User-defined Curl option reset. 996 | */ 997 | Datum http_reset_curlopt(PG_FUNCTION_ARGS); 998 | PG_FUNCTION_INFO_V1(http_reset_curlopt); 999 | Datum http_reset_curlopt(PG_FUNCTION_ARGS) 1000 | { 1001 | http_curlopt *opt = settable_curlopts; 1002 | /* Set up global HTTP handle */ 1003 | CURL * handle = http_get_handle(); 1004 | curl_easy_reset(handle); 1005 | 1006 | /* Clean out the settable_curlopts global cache */ 1007 | while (opt->curlopt) 1008 | { 1009 | if (opt->curlopt_val) guc_free(opt->curlopt_val); 1010 | opt->curlopt_val = NULL; 1011 | opt++; 1012 | } 1013 | 1014 | PG_RETURN_BOOL(true); 1015 | } 1016 | 1017 | Datum http_list_curlopt(PG_FUNCTION_ARGS); 1018 | PG_FUNCTION_INFO_V1(http_list_curlopt); 1019 | Datum http_list_curlopt(PG_FUNCTION_ARGS) 1020 | { 1021 | struct list_state { 1022 | size_t i; /* read position */ 1023 | }; 1024 | 1025 | MemoryContext oldcontext, newcontext; 1026 | FuncCallContext *funcctx; 1027 | struct list_state *state; 1028 | Datum vals[2]; 1029 | bool nulls[2]; 1030 | 1031 | if (SRF_IS_FIRSTCALL()) 1032 | { 1033 | funcctx = SRF_FIRSTCALL_INIT(); 1034 | newcontext = funcctx->multi_call_memory_ctx; 1035 | oldcontext = MemoryContextSwitchTo(newcontext); 1036 | state = palloc0(sizeof(*state)); 1037 | funcctx->user_fctx = state; 1038 | if(get_call_result_type(fcinfo, 0, &funcctx->tuple_desc) != TYPEFUNC_COMPOSITE) 1039 | ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), 1040 | errmsg("composite-returning function called in context that cannot accept a composite"))); 1041 | 1042 | BlessTupleDesc(funcctx->tuple_desc); 1043 | MemoryContextSwitchTo(oldcontext); 1044 | } 1045 | 1046 | funcctx = SRF_PERCALL_SETUP(); 1047 | state = funcctx->user_fctx; 1048 | 1049 | while (1) 1050 | { 1051 | Datum result; 1052 | HeapTuple tuple; 1053 | text *option, *value; 1054 | http_curlopt *opt = settable_curlopts + state->i++; 1055 | if (!opt->curlopt_str) 1056 | break; 1057 | if (!opt->curlopt_val) 1058 | continue; 1059 | option = cstring_to_text(opt->curlopt_str); 1060 | value = cstring_to_text(opt->curlopt_val); 1061 | vals[0] = PointerGetDatum(option); 1062 | vals[1] = PointerGetDatum(value); 1063 | nulls[0] = nulls[1] = 0; 1064 | tuple = heap_form_tuple(funcctx->tuple_desc, vals, nulls); 1065 | result = HeapTupleGetDatum(tuple); 1066 | SRF_RETURN_NEXT(funcctx, result); 1067 | } 1068 | 1069 | SRF_RETURN_DONE(funcctx); 1070 | } 1071 | 1072 | /** 1073 | * User-defined Curl option handling. 1074 | */ 1075 | Datum http_set_curlopt(PG_FUNCTION_ARGS); 1076 | PG_FUNCTION_INFO_V1(http_set_curlopt); 1077 | Datum http_set_curlopt(PG_FUNCTION_ARGS) 1078 | { 1079 | char *curlopt, *value; 1080 | text *curlopt_txt, *value_txt; 1081 | CURL *handle; 1082 | http_curlopt *opt = settable_curlopts; 1083 | 1084 | /* Version check */ 1085 | http_check_curl_version(curl_version_info(CURLVERSION_NOW)); 1086 | 1087 | /* We cannot handle null arguments */ 1088 | if ( PG_ARGISNULL(0) || PG_ARGISNULL(1) ) 1089 | PG_RETURN_BOOL(false); 1090 | 1091 | /* Set up global HTTP handle */ 1092 | handle = http_get_handle(); 1093 | 1094 | /* Read arguments */ 1095 | curlopt_txt = PG_GETARG_TEXT_P(0); 1096 | value_txt = PG_GETARG_TEXT_P(1); 1097 | curlopt = text_to_cstring(curlopt_txt); 1098 | value = text_to_cstring(value_txt); 1099 | 1100 | while (opt->curlopt) 1101 | { 1102 | if (strcasecmp(opt->curlopt_str, curlopt) == 0) 1103 | { 1104 | if (opt->curlopt_val) guc_free(opt->curlopt_val); 1105 | opt->curlopt_val = guc_strdup(ERROR, value); 1106 | PG_RETURN_BOOL(set_curlopt(handle, opt)); 1107 | } 1108 | opt++; 1109 | } 1110 | 1111 | elog(ERROR, "curl option '%s' is not available for run-time configuration", curlopt); 1112 | PG_RETURN_BOOL(false); 1113 | } 1114 | 1115 | 1116 | /** 1117 | * Master HTTP request function, takes in an http_request tuple and outputs 1118 | * an http_response tuple. 1119 | */ 1120 | Datum http_request(PG_FUNCTION_ARGS); 1121 | PG_FUNCTION_INFO_V1(http_request); 1122 | Datum http_request(PG_FUNCTION_ARGS) 1123 | { 1124 | /* Input */ 1125 | HeapTupleHeader rec; 1126 | HeapTupleData tuple; 1127 | Oid tup_type; 1128 | int32 tup_typmod; 1129 | TupleDesc tup_desc; 1130 | int ncolumns; 1131 | Datum *values; 1132 | bool *nulls; 1133 | 1134 | char *uri; 1135 | char *method_str; 1136 | http_method method; 1137 | 1138 | /* Processing */ 1139 | CURLcode err; 1140 | char http_error_buffer[CURL_ERROR_SIZE] = "\0"; 1141 | 1142 | struct curl_slist *headers = NULL; 1143 | StringInfoData si_data; 1144 | StringInfoData si_headers; 1145 | StringInfoData si_read; 1146 | 1147 | int http_return; 1148 | long long_status; 1149 | int status; 1150 | char *content_type = NULL; 1151 | int content_charset = -1; 1152 | 1153 | /* Output */ 1154 | HeapTuple tuple_out; 1155 | 1156 | /* Version check */ 1157 | http_check_curl_version(curl_version_info(CURLVERSION_NOW)); 1158 | 1159 | /* We cannot handle a null request */ 1160 | if ( ! PG_ARGISNULL(0) ) 1161 | rec = PG_GETARG_HEAPTUPLEHEADER(0); 1162 | else 1163 | { 1164 | elog(ERROR, "An http_request must be provided"); 1165 | PG_RETURN_NULL(); 1166 | } 1167 | 1168 | /************************************************************************* 1169 | * Build and run a curl request from the http_request argument 1170 | *************************************************************************/ 1171 | 1172 | /* Zero out static memory */ 1173 | memset(http_error_buffer, 0, sizeof(http_error_buffer)); 1174 | 1175 | /* Extract type info from the tuple itself */ 1176 | tup_type = HeapTupleHeaderGetTypeId(rec); 1177 | tup_typmod = HeapTupleHeaderGetTypMod(rec); 1178 | tup_desc = lookup_rowtype_tupdesc(tup_type, tup_typmod); 1179 | ncolumns = tup_desc->natts; 1180 | 1181 | /* Build a temporary HeapTuple control structure */ 1182 | tuple.t_len = HeapTupleHeaderGetDatumLength(rec); 1183 | ItemPointerSetInvalid(&(tuple.t_self)); 1184 | tuple.t_tableOid = InvalidOid; 1185 | tuple.t_data = rec; 1186 | 1187 | /* Prepare for values / nulls */ 1188 | values = (Datum *) palloc0(ncolumns * sizeof(Datum)); 1189 | nulls = (bool *) palloc0(ncolumns * sizeof(bool)); 1190 | 1191 | /* Break down the tuple into values/nulls lists */ 1192 | heap_deform_tuple(&tuple, tup_desc, values, nulls); 1193 | 1194 | /* Read the URI */ 1195 | if ( nulls[REQ_URI] ) 1196 | elog(ERROR, "http_request.uri is NULL"); 1197 | uri = TextDatumGetCString(values[REQ_URI]); 1198 | 1199 | /* Read the method */ 1200 | if ( nulls[REQ_METHOD] ) 1201 | elog(ERROR, "http_request.method is NULL"); 1202 | method_str = TextDatumGetCString(values[REQ_METHOD]); 1203 | method = request_type(method_str); 1204 | elog(DEBUG2, "pgsql-http: method_str: '%s', method: %d", method_str, method); 1205 | 1206 | /* Set up global HTTP handle */ 1207 | g_http_handle = http_get_handle(); 1208 | 1209 | /* Set up the error buffer */ 1210 | CURL_SETOPT(g_http_handle, CURLOPT_ERRORBUFFER, http_error_buffer); 1211 | 1212 | /* Set the target URL */ 1213 | CURL_SETOPT(g_http_handle, CURLOPT_URL, uri); 1214 | 1215 | 1216 | /* Restrict to just http/https. Leaving unrestricted */ 1217 | /* opens possibility of users requesting file:/// urls */ 1218 | /* locally */ 1219 | #if LIBCURL_VERSION_NUM >= 0x075400 /* 7.84.0 */ 1220 | CURL_SETOPT(g_http_handle, CURLOPT_PROTOCOLS_STR, "http,https"); 1221 | #else 1222 | CURL_SETOPT(g_http_handle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); 1223 | #endif 1224 | 1225 | if ( curlopt_is_set(CURLOPT_TCP_KEEPALIVE) ) 1226 | { 1227 | /* Keep sockets held open */ 1228 | CURL_SETOPT(g_http_handle, CURLOPT_FORBID_REUSE, 0); 1229 | } 1230 | else 1231 | { 1232 | /* Keep sockets from being held open */ 1233 | CURL_SETOPT(g_http_handle, CURLOPT_FORBID_REUSE, 1); 1234 | } 1235 | 1236 | /* Set up the write-back function */ 1237 | CURL_SETOPT(g_http_handle, CURLOPT_WRITEFUNCTION, http_writeback); 1238 | 1239 | /* Set up the write-back buffer */ 1240 | initStringInfo(&si_data); 1241 | initStringInfo(&si_headers); 1242 | CURL_SETOPT(g_http_handle, CURLOPT_WRITEDATA, (void*)(&si_data)); 1243 | CURL_SETOPT(g_http_handle, CURLOPT_WRITEHEADER, (void*)(&si_headers)); 1244 | 1245 | #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ 1246 | /* Connect the progress callback for interrupt support */ 1247 | CURL_SETOPT(g_http_handle, CURLOPT_XFERINFOFUNCTION, http_progress_callback); 1248 | CURL_SETOPT(g_http_handle, CURLOPT_NOPROGRESS, 0); 1249 | #endif 1250 | 1251 | /* Set the HTTP content encoding to all curl supports */ 1252 | CURL_SETOPT(g_http_handle, CURLOPT_ACCEPT_ENCODING, ""); 1253 | 1254 | if ( method != HTTP_HEAD ) 1255 | { 1256 | /* Follow redirects, as many as 5 */ 1257 | CURL_SETOPT(g_http_handle, CURLOPT_FOLLOWLOCATION, 1); 1258 | CURL_SETOPT(g_http_handle, CURLOPT_MAXREDIRS, 5); 1259 | } 1260 | 1261 | if ( curlopt_is_set(CURLOPT_TCP_KEEPALIVE) ) 1262 | { 1263 | /* Add a keep alive option to the headers to reuse network sockets */ 1264 | headers = curl_slist_append(headers, "Connection: Keep-Alive"); 1265 | } 1266 | else 1267 | { 1268 | /* Add a close option to the headers to avoid open network sockets */ 1269 | headers = curl_slist_append(headers, "Connection: close"); 1270 | } 1271 | 1272 | /* Let our charset preference be known */ 1273 | headers = curl_slist_append(headers, "Charsets: utf-8"); 1274 | 1275 | /* Handle optional headers */ 1276 | if ( ! nulls[REQ_HEADERS] ) 1277 | { 1278 | ArrayType *array = DatumGetArrayTypeP(values[REQ_HEADERS]); 1279 | headers = header_array_to_slist(array, headers); 1280 | } 1281 | 1282 | /* If we have a payload we send it, assuming we're either POST, GET, PATCH, PUT or DELETE or UNKNOWN */ 1283 | if ( ! nulls[REQ_CONTENT] && values[REQ_CONTENT] ) 1284 | { 1285 | text *content_text; 1286 | long content_size; 1287 | char *cstr; 1288 | char buffer[1024]; 1289 | 1290 | /* Read the content type */ 1291 | if ( nulls[REQ_CONTENT_TYPE] || ! values[REQ_CONTENT_TYPE] ) 1292 | elog(ERROR, "http_request.content_type is NULL"); 1293 | cstr = TextDatumGetCString(values[REQ_CONTENT_TYPE]); 1294 | 1295 | /* Add content type to the headers */ 1296 | snprintf(buffer, sizeof(buffer), "Content-Type: %s", cstr); 1297 | headers = curl_slist_append(headers, buffer); 1298 | pfree(cstr); 1299 | 1300 | /* Read the content */ 1301 | content_text = DatumGetTextP(values[REQ_CONTENT]); 1302 | content_size = VARSIZE_ANY_EXHDR(content_text); 1303 | 1304 | if ( method == HTTP_GET || method == HTTP_POST || method == HTTP_DELETE ) 1305 | { 1306 | /* Add the content to the payload */ 1307 | CURL_SETOPT(g_http_handle, CURLOPT_POST, 1); 1308 | if ( method == HTTP_GET ) 1309 | { 1310 | /* Force the verb to be GET */ 1311 | CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "GET"); 1312 | } 1313 | else if( method == HTTP_DELETE ) 1314 | { 1315 | /* Force the verb to be DELETE */ 1316 | CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "DELETE"); 1317 | } 1318 | 1319 | CURL_SETOPT(g_http_handle, CURLOPT_POSTFIELDS, (char *)(VARDATA(content_text))); 1320 | CURL_SETOPT(g_http_handle, CURLOPT_POSTFIELDSIZE, content_size); 1321 | } 1322 | else if ( method == HTTP_PUT || method == HTTP_PATCH || method == HTTP_UNKNOWN ) 1323 | { 1324 | if ( method == HTTP_PATCH ) 1325 | CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "PATCH"); 1326 | 1327 | /* Assume the user knows what they are doing and pass unchanged */ 1328 | if ( method == HTTP_UNKNOWN ) 1329 | CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, method_str); 1330 | 1331 | initStringInfo(&si_read); 1332 | appendBinaryStringInfo(&si_read, VARDATA(content_text), content_size); 1333 | CURL_SETOPT(g_http_handle, CURLOPT_UPLOAD, 1); 1334 | CURL_SETOPT(g_http_handle, CURLOPT_READFUNCTION, http_readback); 1335 | CURL_SETOPT(g_http_handle, CURLOPT_READDATA, &si_read); 1336 | CURL_SETOPT(g_http_handle, CURLOPT_INFILESIZE, content_size); 1337 | } 1338 | else 1339 | { 1340 | /* Never get here */ 1341 | elog(ERROR, "illegal HTTP method"); 1342 | } 1343 | } 1344 | else if ( method == HTTP_DELETE ) 1345 | { 1346 | CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, "DELETE"); 1347 | } 1348 | else if ( method == HTTP_HEAD ) 1349 | { 1350 | CURL_SETOPT(g_http_handle, CURLOPT_NOBODY, 1); 1351 | } 1352 | else if ( method == HTTP_PUT || method == HTTP_POST ) 1353 | { 1354 | /* If we had a content we do not reach that part */ 1355 | elog(ERROR, "http_request.content is NULL"); 1356 | } 1357 | else if ( method == HTTP_UNKNOWN ){ 1358 | /* Assume the user knows what they are doing and pass unchanged */ 1359 | CURL_SETOPT(g_http_handle, CURLOPT_CUSTOMREQUEST, method_str); 1360 | } 1361 | 1362 | pfree(method_str); 1363 | /* Set the headers */ 1364 | CURL_SETOPT(g_http_handle, CURLOPT_HTTPHEADER, headers); 1365 | 1366 | /************************************************************************* 1367 | * PERFORM THE REQUEST! 1368 | **************************************************************************/ 1369 | http_return = curl_easy_perform(g_http_handle); 1370 | elog(DEBUG2, "pgsql-http: queried '%s'", uri); 1371 | elog(DEBUG2, "pgsql-http: http_return '%d'", http_return); 1372 | 1373 | /* Clean up some input things we don't need anymore */ 1374 | ReleaseTupleDesc(tup_desc); 1375 | pfree(values); 1376 | pfree(nulls); 1377 | 1378 | /************************************************************************* 1379 | * Create an http_response object from the curl results 1380 | *************************************************************************/ 1381 | 1382 | /* Write out an error on failure */ 1383 | if ( http_return != CURLE_OK ) 1384 | { 1385 | curl_slist_free_all(headers); 1386 | curl_easy_cleanup(g_http_handle); 1387 | g_http_handle = NULL; 1388 | 1389 | #if LIBCURL_VERSION_NUM >= 0x072700 /* 7.39.0 */ 1390 | /* 1391 | * If the request was aborted by an interrupt request 1392 | * report back. 1393 | */ 1394 | if (http_return == CURLE_ABORTED_BY_CALLBACK) 1395 | elog(ERROR, "canceling statement due to user request"); 1396 | #endif 1397 | 1398 | http_error(http_return, http_error_buffer); 1399 | } 1400 | 1401 | /* Read the metadata from the handle directly */ 1402 | if ( (CURLE_OK != curl_easy_getinfo(g_http_handle, CURLINFO_RESPONSE_CODE, &long_status)) || 1403 | (CURLE_OK != curl_easy_getinfo(g_http_handle, CURLINFO_CONTENT_TYPE, &content_type)) ) 1404 | { 1405 | curl_slist_free_all(headers); 1406 | curl_easy_cleanup(g_http_handle); 1407 | g_http_handle = NULL; 1408 | ereport(ERROR, (errmsg("CURL: Error in curl_easy_getinfo"))); 1409 | } 1410 | 1411 | /* Prepare our return object */ 1412 | if (get_call_result_type(fcinfo, 0, &tup_desc) != TYPEFUNC_COMPOSITE) { 1413 | ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), 1414 | errmsg("%s called with incompatible return type", __func__))); 1415 | } 1416 | 1417 | ncolumns = tup_desc->natts; 1418 | values = palloc0(sizeof(Datum)*ncolumns); 1419 | nulls = palloc0(sizeof(bool)*ncolumns); 1420 | 1421 | /* Status code */ 1422 | status = long_status; 1423 | values[RESP_STATUS] = Int32GetDatum(status); 1424 | nulls[RESP_STATUS] = false; 1425 | 1426 | /* Content type */ 1427 | if ( content_type ) 1428 | { 1429 | List *ctl; 1430 | ListCell *lc; 1431 | 1432 | values[RESP_CONTENT_TYPE] = CStringGetTextDatum(content_type); 1433 | nulls[RESP_CONTENT_TYPE] = false; 1434 | 1435 | /* Read the character set name out of the content type */ 1436 | /* if there is one in there */ 1437 | /* text/html; charset=iso-8859-1 */ 1438 | if ( SplitIdentifierString(pstrdup(content_type), ';', &ctl) ) 1439 | { 1440 | foreach(lc, ctl) 1441 | { 1442 | /* charset=iso-8859-1 */ 1443 | const char *param = (const char *) lfirst(lc); 1444 | const char *paramtype = "charset="; 1445 | if ( http_strcasestr(param, paramtype) ) 1446 | { 1447 | /* iso-8859-1 */ 1448 | const char *charset = param + strlen(paramtype); 1449 | content_charset = pg_char_to_encoding(charset); 1450 | break; 1451 | } 1452 | } 1453 | } 1454 | } 1455 | else 1456 | { 1457 | values[RESP_CONTENT_TYPE] = (Datum)0; 1458 | nulls[RESP_CONTENT_TYPE] = true; 1459 | } 1460 | 1461 | /* Headers array */ 1462 | if ( si_headers.len ) 1463 | { 1464 | /* Strip the carriage-returns, because who cares? */ 1465 | string_info_remove_cr(&si_headers); 1466 | values[RESP_HEADERS] = PointerGetDatum(header_string_to_array(&si_headers)); 1467 | nulls[RESP_HEADERS] = false; 1468 | } 1469 | else 1470 | { 1471 | values[RESP_HEADERS] = (Datum)0; 1472 | nulls[RESP_HEADERS] = true; 1473 | } 1474 | 1475 | /* Content */ 1476 | if ( si_data.len ) 1477 | { 1478 | char *content_str; 1479 | size_t content_len; 1480 | elog(DEBUG2, "pgsql-http: content_charset = %d", content_charset); 1481 | 1482 | /* Apply character transcoding if necessary */ 1483 | if ( content_charset < 0 ) 1484 | { 1485 | content_str = si_data.data; 1486 | content_len = si_data.len; 1487 | } 1488 | else 1489 | { 1490 | content_str = pg_any_to_server(si_data.data, si_data.len, content_charset); 1491 | content_len = strlen(content_str); 1492 | } 1493 | 1494 | values[RESP_CONTENT] = PointerGetDatum(cstring_to_text_with_len(content_str, content_len)); 1495 | nulls[RESP_CONTENT] = false; 1496 | } 1497 | else 1498 | { 1499 | values[RESP_CONTENT] = (Datum)0; 1500 | nulls[RESP_CONTENT] = true; 1501 | } 1502 | 1503 | /* Build up a tuple from values/nulls lists */ 1504 | tuple_out = heap_form_tuple(tup_desc, values, nulls); 1505 | 1506 | /* Clean up */ 1507 | ReleaseTupleDesc(tup_desc); 1508 | if ( ! curlopt_is_set(CURLOPT_TCP_KEEPALIVE) ) 1509 | { 1510 | curl_easy_cleanup(g_http_handle); 1511 | g_http_handle = NULL; 1512 | } 1513 | curl_slist_free_all(headers); 1514 | pfree(si_headers.data); 1515 | pfree(si_data.data); 1516 | pfree(values); 1517 | pfree(nulls); 1518 | 1519 | /* Return */ 1520 | PG_RETURN_DATUM(HeapTupleGetDatum(tuple_out)); 1521 | } 1522 | 1523 | 1524 | 1525 | 1526 | /* URL Encode Escape Chars */ 1527 | /* 45-46 (-.) 48-57 (0-9) 65-90 (A-Z) */ 1528 | /* 95 (_) 97-122 (a-z) 126 (~) */ 1529 | 1530 | static int chars_to_not_encode[] = { 1531 | 0,0,0,0,0,0,0,0,0,0, 1532 | 0,0,0,0,0,0,0,0,0,0, 1533 | 0,0,0,0,0,0,0,0,0,0, 1534 | 0,0,0,0,0,0,0,0,0,0, 1535 | 0,0,0,0,0,1,1,0,1,1, 1536 | 1,1,1,1,1,1,1,1,0,0, 1537 | 0,0,0,0,0,1,1,1,1,1, 1538 | 1,1,1,1,1,1,1,1,1,1, 1539 | 1,1,1,1,1,1,1,1,1,1, 1540 | 1,0,0,0,0,1,0,1,1,1, 1541 | 1,1,1,1,1,1,1,1,1,1, 1542 | 1,1,1,1,1,1,1,1,1,1, 1543 | 1,1,1,0,0,0,1,0 1544 | }; 1545 | 1546 | 1547 | 1548 | /* 1549 | * Take in a text pointer and output a cstring with 1550 | * all encodable characters encoded. 1551 | */ 1552 | static char* 1553 | urlencode_cstr(const char* str_in, size_t str_in_len) 1554 | { 1555 | char *str_out, *ptr; 1556 | size_t i; 1557 | int rv; 1558 | 1559 | if (!str_in_len) return pstrdup(""); 1560 | 1561 | /* Prepare the output string, encoding can fluff the ouput */ 1562 | /* considerably */ 1563 | str_out = palloc0(str_in_len * 4); 1564 | ptr = str_out; 1565 | 1566 | for (i = 0; i < str_in_len; i++) 1567 | { 1568 | unsigned char c = str_in[i]; 1569 | 1570 | /* Break on NULL */ 1571 | if (c == '\0') 1572 | break; 1573 | 1574 | /* Replace ' ' with '+' */ 1575 | if (c == ' ') 1576 | { 1577 | *ptr = '+'; 1578 | ptr++; 1579 | continue; 1580 | } 1581 | 1582 | /* Pass basic characters through */ 1583 | if ((c < 127) && chars_to_not_encode[(int)(str_in[i])]) 1584 | { 1585 | *ptr = str_in[i]; 1586 | ptr++; 1587 | continue; 1588 | } 1589 | 1590 | /* Encode the remaining chars */ 1591 | rv = snprintf(ptr, 4, "%%%02X", c); 1592 | if ( rv < 0 ) 1593 | return NULL; 1594 | 1595 | /* Move pointer forward */ 1596 | ptr += 3; 1597 | } 1598 | *ptr = '\0'; 1599 | 1600 | return str_out; 1601 | } 1602 | 1603 | /** 1604 | * Utility function for users building URL encoded requests, applies 1605 | * standard URL encoding to an input string. 1606 | */ 1607 | Datum urlencode(PG_FUNCTION_ARGS); 1608 | PG_FUNCTION_INFO_V1(urlencode); 1609 | Datum urlencode(PG_FUNCTION_ARGS) 1610 | { 1611 | /* Declare SQL function strict, so no test for NULL input */ 1612 | text *txt = PG_GETARG_TEXT_P(0); 1613 | char *encoded = urlencode_cstr(VARDATA(txt), VARSIZE_ANY_EXHDR(txt)); 1614 | if (encoded) 1615 | PG_RETURN_TEXT_P(cstring_to_text(encoded)); 1616 | else 1617 | PG_RETURN_NULL(); 1618 | } 1619 | 1620 | /** 1621 | * Treat the top level jsonb map as a key/value set 1622 | * to be fed into urlencode and return a correctly 1623 | * encoded data string. 1624 | */ 1625 | Datum urlencode_jsonb(PG_FUNCTION_ARGS); 1626 | PG_FUNCTION_INFO_V1(urlencode_jsonb); 1627 | Datum urlencode_jsonb(PG_FUNCTION_ARGS) 1628 | { 1629 | bool skipNested = false; 1630 | Jsonb* jb = PG_GETARG_JSONB_P(0); 1631 | JsonbIterator *it; 1632 | JsonbValue v; 1633 | JsonbIteratorToken r; 1634 | StringInfoData si; 1635 | size_t count = 0; 1636 | 1637 | if (!JB_ROOT_IS_OBJECT(jb)) 1638 | { 1639 | ereport(ERROR, 1640 | (errcode(ERRCODE_INVALID_PARAMETER_VALUE), 1641 | errmsg("cannot call %s on a non-object", __func__))); 1642 | } 1643 | 1644 | /* Buffer to write complete output into */ 1645 | initStringInfo(&si); 1646 | 1647 | it = JsonbIteratorInit(&jb->root); 1648 | while ((r = JsonbIteratorNext(&it, &v, skipNested)) != WJB_DONE) 1649 | { 1650 | skipNested = true; 1651 | 1652 | if (r == WJB_KEY) 1653 | { 1654 | char *key, *key_enc, *value, *value_enc; 1655 | 1656 | /* Skip zero-length key */ 1657 | if(!v.val.string.len) continue; 1658 | 1659 | /* Read and encode the key */ 1660 | key = pnstrdup(v.val.string.val, v.val.string.len); 1661 | key_enc = urlencode_cstr(v.val.string.val, v.val.string.len); 1662 | 1663 | /* Read the value for this key */ 1664 | #if PG_VERSION_NUM < 130000 1665 | { 1666 | JsonbValue k; 1667 | k.type = jbvString; 1668 | k.val.string.val = key; 1669 | k.val.string.len = strlen(key); 1670 | v = *findJsonbValueFromContainer(&jb->root, JB_FOBJECT, &k); 1671 | } 1672 | #else 1673 | getKeyJsonValueFromContainer(&jb->root, key, strlen(key), &v); 1674 | #endif 1675 | /* Read and encode the value */ 1676 | switch(v.type) 1677 | { 1678 | case jbvString: { 1679 | value = pnstrdup(v.val.string.val, v.val.string.len); 1680 | break; 1681 | } 1682 | case jbvNumeric: { 1683 | value = numeric_normalize(v.val.numeric); 1684 | break; 1685 | } 1686 | case jbvBool: { 1687 | value = pstrdup(v.val.boolean ? "true" : "false"); 1688 | break; 1689 | } 1690 | case jbvNull: { 1691 | value = pstrdup(""); 1692 | break; 1693 | } 1694 | default: { 1695 | elog(DEBUG2, "skipping non-scalar value of '%s'", key); 1696 | continue; 1697 | } 1698 | 1699 | } 1700 | /* Write the result */ 1701 | value_enc = urlencode_cstr(value, strlen(value)); 1702 | if (count++) appendStringInfo(&si, "&"); 1703 | appendStringInfo(&si, "%s=%s", key_enc, value_enc); 1704 | 1705 | /* Clean up temporary strings */ 1706 | if (key) pfree(key); 1707 | if (value) pfree(value); 1708 | if (key_enc) pfree(key_enc); 1709 | if (value_enc) pfree(value_enc); 1710 | } 1711 | } 1712 | 1713 | if (si.len) 1714 | PG_RETURN_TEXT_P(cstring_to_text_with_len(si.data, si.len)); 1715 | else 1716 | PG_RETURN_NULL(); 1717 | } 1718 | 1719 | Datum bytea_to_text(PG_FUNCTION_ARGS); 1720 | PG_FUNCTION_INFO_V1(bytea_to_text); 1721 | Datum bytea_to_text(PG_FUNCTION_ARGS) 1722 | { 1723 | bytea *b = PG_GETARG_BYTEA_P(0); 1724 | text *t = palloc(VARSIZE_ANY(b)); 1725 | memcpy(t, b, VARSIZE(b)); 1726 | PG_RETURN_TEXT_P(t); 1727 | } 1728 | 1729 | Datum text_to_bytea(PG_FUNCTION_ARGS); 1730 | PG_FUNCTION_INFO_V1(text_to_bytea); 1731 | Datum text_to_bytea(PG_FUNCTION_ARGS) 1732 | { 1733 | text *t = PG_GETARG_TEXT_P(0); 1734 | bytea *b = palloc(VARSIZE_ANY(t)); 1735 | memcpy(b, t, VARSIZE(t)); 1736 | PG_RETURN_TEXT_P(b); 1737 | } 1738 | 1739 | 1740 | // Local Variables: 1741 | // mode: C++ 1742 | // tab-width: 4 1743 | // c-basic-offset: 4 1744 | // indent-tabs-mode: t 1745 | // End: 1746 | --------------------------------------------------------------------------------