├── META.json ├── Makefile ├── README.md ├── connutil.c ├── connutil.h ├── doc └── s3_fdw.md ├── s3_fdw.c ├── s3_fdw.control └── s3_fdw.sql /META.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "s3_fdw", 3 | "abstract": "foreign-data wrapper for Amazon S3", 4 | "description": "This module provides reading files located in Amazon S3 privately, using COPY mechanism.", 5 | "version": "0.1.0", 6 | "maintainer": "Hitoshi Harada ", 7 | "license": "postgresql", 8 | "provides": { 9 | "s3_fdw": { 10 | "abstract": "fdw for Amazon S3", 11 | "version": "0.1.0", 12 | "file": "s3_fdw.sql", 13 | "docfile": "doc/s3_fdw.md" 14 | } 15 | }, 16 | "resources": { 17 | "bugtracker": { 18 | "web": "http://github.com/umitanuki/s3_fdw/issues/" 19 | }, 20 | "repository": { 21 | "url": "git://github.com/umitanuki/s3_fdw.git", 22 | "web": "http://github.com/umitanuki/s3_fdw", 23 | "type": "git" 24 | } 25 | }, 26 | "release_status": "unstable", 27 | "meta-spec": { 28 | "version": "1.0.0", 29 | "url": "http://pgxn.org/meta/spec.txt" 30 | }, 31 | "tags": [ 32 | "fdw", 33 | "web", 34 | "internet", 35 | "amazon", 36 | "cloud", 37 | "bulkload" 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | MODULE_big = s3_fdw 3 | OBJS = s3_fdw.o connutil.o# copy_patched.o 4 | EXTENSION = $(MODULE_big) 5 | EXTVERSION = 0.1.0 6 | EXTSQL = $(MODULE_big)--$(EXTVERSION).sql 7 | DATA = $(EXTSQL) 8 | EXTRA_CLEAN += $(EXTSQL) 9 | SHLIB_LINK = -lcurl -lssl -lcrypto 10 | 11 | #DOCS = doc/$(MODULES).md 12 | REGRESS = $(MODULE_big) 13 | 14 | all: $(EXTSQL) 15 | 16 | $(EXTSQL): $(MODULE_big).sql 17 | cp $< $@ 18 | 19 | 20 | 21 | PG_CONFIG = pg_config 22 | PGXS := $(shell $(PG_CONFIG) --pgxs) 23 | include $(PGXS) 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | s3\_fdw 2 | ======= 3 | 4 | s3\_fdw provides a foreign-data wrapper (FDW) for Amazon S3 files, 5 | using the builtin COPY format. 6 | 7 | To build it, just do this: 8 | 9 | make 10 | make install 11 | 12 | If you encounter an error such as: 13 | 14 | make: pg_config: Command not found 15 | 16 | Be sure that you have `pg_config` installed and in your path. If you used a 17 | package management system such as RPM to install PostgreSQL, be sure that the 18 | `-devel` package is also installed. If necessary tell the build process where 19 | to find it: 20 | 21 | env PG_CONFIG=/path/to/pg_config make && make installcheck && make install 22 | 23 | Once `make install` is done, connect to your database with psql or other client 24 | and type 25 | 26 | CREATE EXTENSION s3_fdw; 27 | 28 | then you'll see the FDW is installed. With the FDW, create server, user mapping, 29 | foreign table. You'll need Amazon S3 access key ID and secret access key to 30 | authenticate private access to your data. Consult AWS documentation for those keys. 31 | The access information is stored in user mapping. Foreign tables stores options 32 | for COPY as well as hostname, bucketname and filename. 33 | 34 | Dependencies 35 | ------------ 36 | The `s3_fdw` data type depends on libcurl and openssl. You need those developer 37 | packages installed in the system path. 38 | -------------------------------------------------------------------------------- /connutil.c: -------------------------------------------------------------------------------- 1 | #include "openssl/hmac.h" 2 | 3 | #include "postgres.h" 4 | #include "lib/stringinfo.h" 5 | 6 | #include "connutil.h" 7 | 8 | static char *sign_by_secretkey(char *input, char *secretkey); 9 | static int b64_encode(const uint8 *src, unsigned len, uint8 *dst); 10 | 11 | /* 12 | * Constructs GMT-style string 13 | */ 14 | char * 15 | httpdate(time_t *timer) 16 | { 17 | char *datestring; 18 | time_t t; 19 | struct tm *gt; 20 | 21 | t = time(timer); 22 | gt = gmtime(&t); 23 | datestring = (char *) palloc0(256 * sizeof(char)); 24 | strftime(datestring, 256 * sizeof(char), "%a, %d %b %Y %H:%M:%S +0000", gt); 25 | return datestring; 26 | } 27 | 28 | /* 29 | * Construct signed string for the Authorization header, 30 | * following the Amazon S3 REST API spec. 31 | */ 32 | char * 33 | s3_signature(char *method, char *datestring, 34 | char *bucket, char *file, char *secretkey) 35 | { 36 | size_t rs_size; 37 | char *resource; 38 | StringInfoData buf; 39 | 40 | rs_size = strlen(bucket) + strlen(file) + 3; /* 3 = '/' + '/' + '\0' */ 41 | resource = (char *) palloc0(rs_size); 42 | 43 | snprintf(resource, rs_size, "/%s/%s", bucket, file); 44 | initStringInfo(&buf); 45 | /* 46 | * StringToSign = HTTP-Verb + "\n" + 47 | * Content-MD5 + "\n" + 48 | * Content-Type + "\n" + 49 | * Date + "\n" + 50 | * CanonicalizedAmzHeaders + 51 | * CanonicalizedResource; 52 | */ 53 | appendStringInfo(&buf, "%s\n", method); 54 | appendStringInfo(&buf, "\n"); 55 | appendStringInfo(&buf, "\n"); 56 | appendStringInfo(&buf, "%s\n", datestring); 57 | // appendStringInfo(&buf, ""); 58 | appendStringInfo(&buf, "%s", resource); 59 | 60 | //elog(INFO, "StringToSign:%s", buf.data); 61 | return sign_by_secretkey(buf.data, secretkey); 62 | } 63 | 64 | static char * 65 | sign_by_secretkey(char *input, char *secretkey) 66 | { 67 | HMAC_CTX ctx; 68 | /* sha1 has to be 30 charcters */ 69 | char result[256]; 70 | unsigned int len; 71 | /* base64 may enlarge the size up to double */ 72 | char b64_result[256]; 73 | int b64_len; 74 | 75 | HMAC_CTX_init(&ctx); 76 | HMAC_Init(&ctx, secretkey, strlen(secretkey), EVP_sha1()); 77 | HMAC_Update(&ctx, (unsigned char *) input, strlen(input)); 78 | HMAC_Final(&ctx, (unsigned char *) result, &len); 79 | HMAC_CTX_cleanup(&ctx); 80 | 81 | b64_len = b64_encode((unsigned char *) result, len, (unsigned char *) b64_result); 82 | b64_result[b64_len] = '\0'; 83 | 84 | return pstrdup(b64_result); 85 | } 86 | 87 | /* 88 | * BASE64 - duplicated :( 89 | */ 90 | 91 | static int 92 | b64_encode(const uint8 *src, unsigned len, uint8 *dst) 93 | { 94 | static const unsigned char _base64[] = 95 | "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 96 | 97 | uint8 *p, 98 | *lend = dst + 76; 99 | const uint8 *s, 100 | *end = src + len; 101 | int pos = 2; 102 | unsigned long buf = 0; 103 | 104 | s = src; 105 | p = dst; 106 | 107 | while (s < end) 108 | { 109 | buf |= *s << (pos << 3); 110 | pos--; 111 | s++; 112 | 113 | /* 114 | * write it out 115 | */ 116 | if (pos < 0) 117 | { 118 | *p++ = _base64[(buf >> 18) & 0x3f]; 119 | *p++ = _base64[(buf >> 12) & 0x3f]; 120 | *p++ = _base64[(buf >> 6) & 0x3f]; 121 | *p++ = _base64[buf & 0x3f]; 122 | 123 | pos = 2; 124 | buf = 0; 125 | } 126 | if (p >= lend) 127 | { 128 | *p++ = '\n'; 129 | lend = p + 76; 130 | } 131 | } 132 | if (pos != 2) 133 | { 134 | *p++ = _base64[(buf >> 18) & 0x3f]; 135 | *p++ = _base64[(buf >> 12) & 0x3f]; 136 | *p++ = (pos == 0) ? _base64[(buf >> 6) & 0x3f] : '='; 137 | *p++ = '='; 138 | } 139 | 140 | return p - dst; 141 | } 142 | -------------------------------------------------------------------------------- /connutil.h: -------------------------------------------------------------------------------- 1 | #ifndef _S3_CONNUTIL_H_ 2 | #define _S3_CONNUTIL_H_ 3 | 4 | #include 5 | 6 | extern char *httpdate(time_t *timer); 7 | extern char *s3_signature(char *method, char *datestring, 8 | char *bucket, char *file, char *secretkey); 9 | 10 | 11 | #endif /* _S3_CONNUTIL_H */ 12 | -------------------------------------------------------------------------------- /doc/s3_fdw.md: -------------------------------------------------------------------------------- 1 | s3\_fdw 2 | ======= 3 | 4 | Synopsis 5 | -------- 6 | 7 | db1=# CREATE EXTENSION s3_fdw; 8 | CREATE EXTENSION 9 | 10 | db1=# CREATE SERVER amazon_s3 FOREIGN DATA WRAPPER s3_fdw; 11 | CREATE SERVER 12 | 13 | db1=# CREATE USER MAPPING FOR CURRENT_USER SERVER amazon_s3 14 | OPTIONS ( 15 | accesskey 'your-access-key-id', 16 | secretkey 'your-secret-access-key' 17 | ); 18 | CREATE USER MAPPING 19 | 20 | db1=# CREATE FOREIGN TABLE log20110901( 21 | atime timestamp, 22 | method text, elapse int, 23 | session text 24 | ) SERVER amazon_s3 25 | OPTIONS ( 26 | hostname 's3-ap-northeast-1.amazonaws.com', 27 | bucketname 'umitanuki-dbtest', 28 | filename 'log20110901.txt', 29 | delimiter E'\t' 30 | ); 31 | CREATE FOREIGN TABLE 32 | 33 | Description 34 | ----------- 35 | 36 | This module provides foreign-data wrapper for Amazon S3 files. 37 | The procedure to initiate your foreign table is shown above. 38 | For the first process, `create extension` for this module. Then, 39 | `create server` with some name whatever you like without options, 40 | since server option is not supported yet. After that, 41 | `create user mapping` for current user with mandatory options 42 | `accesskey` and `secretkey`. They are provied from Amazon to you. 43 | 44 | Last, `create foreign table` for your file. At the moment you 45 | need to define one table for one file, as file\_fdw in contrib. 46 | s3\_fdw does support all the COPY options as file\_fdw does, as 47 | well as these additional mandatory options: 48 | 49 | - hostname 50 | - bucketname 51 | - filename 52 | 53 | You'll find the access URL to S3 file. Split it into these 54 | tree options and specify separately. 55 | 56 | Roadmap 57 | ------- 58 | 59 | - gz file support 60 | - bucket files bulk load 61 | - normal URL option rather than split path 62 | - windows support 63 | 64 | Caveat 65 | ------ 66 | 67 | This module is still under development. You may encounter 68 | unpredictable situation by using this program. 69 | 70 | Especially s3\_fdw forks backend and calls mkfifo to achieve 71 | read and write in parallel. So, it doesn't work on the 72 | platforms in which fork / mkfifo doesn't work. 73 | 74 | Support 75 | ------- 76 | 77 | Goto http://github.com/umitanuki/s3_fdw 78 | Feel free to report any bug/issues if you find. 79 | 80 | Author 81 | ------ 82 | 83 | Hitoshi Harada 84 | 85 | -------------------------------------------------------------------------------- /s3_fdw.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "curl/curl.h" 6 | 7 | #include "postgres.h" 8 | #include "fmgr.h" 9 | 10 | #include "access/reloptions.h" 11 | #include "catalog/pg_foreign_table.h" 12 | #include "catalog/pg_user_mapping.h" 13 | #include "commands/copy.h" 14 | #include "commands/defrem.h" 15 | #include "commands/explain.h" 16 | #include "foreign/fdwapi.h" 17 | #include "foreign/foreign.h" 18 | #include "lib/stringinfo.h" 19 | #include "libpq/pqsignal.h" 20 | #include "miscadmin.h" 21 | #include "optimizer/cost.h" 22 | #include "postmaster/fork_process.h" 23 | #include "storage/fd.h" 24 | #include "storage/ipc.h" 25 | #include "utils/builtins.h" 26 | #include "utils/rel.h" 27 | //#include "utils/resowner.h" 28 | 29 | #include "connutil.h" 30 | 31 | PG_MODULE_MAGIC; 32 | 33 | 34 | /* 35 | * Describes the valid options for objects that use this wrapper. 36 | */ 37 | struct S3FdwOption 38 | { 39 | const char *optname; 40 | Oid optcontext; /* Oid of catalog in which option may appear */ 41 | }; 42 | 43 | /* 44 | * Valid options for s3_fdw. 45 | * These options are based on the options for COPY FROM command. 46 | * 47 | * Note: If you are adding new option for user mapping, you need to modify 48 | * s3GetOptions(), which currently doesn't bother to look at user mappings. 49 | */ 50 | static struct S3FdwOption valid_options[] = { 51 | /* File options */ 52 | {"filename", ForeignTableRelationId}, 53 | 54 | {"bucketname", ForeignTableRelationId}, 55 | {"hostname", ForeignTableRelationId}, 56 | 57 | /* Format options */ 58 | /* oids option is not supported */ 59 | {"format", ForeignTableRelationId}, 60 | {"header", ForeignTableRelationId}, 61 | {"delimiter", ForeignTableRelationId}, 62 | {"quote", ForeignTableRelationId}, 63 | {"escape", ForeignTableRelationId}, 64 | {"null", ForeignTableRelationId}, 65 | {"encoding", ForeignTableRelationId}, 66 | 67 | {"accesskey", UserMappingRelationId}, 68 | {"secretkey", UserMappingRelationId}, 69 | 70 | /* Sentinel */ 71 | {NULL, InvalidOid} 72 | }; 73 | 74 | /* 75 | * FDW-specific information for ForeignScanState.fdw_state. 76 | */ 77 | typedef struct S3FdwExecutionState 78 | { 79 | char *hostname; 80 | char *bucketname; 81 | char *filename; /* file to read */ 82 | char *accesskey; 83 | char *secretkey; 84 | List *copy_options; /* merged COPY options, excluding filename */ 85 | CopyState cstate; /* state of reading file */ 86 | char *datafn; 87 | } S3FdwExecutionState; 88 | 89 | /* 90 | * forked processes communicate via FIFO, which is described 91 | * in this struct. Some experiments tell that it should be 92 | * a bad idead to re-open these FIFO; we prepare two files 93 | * as one for synchronizing flag, the other for data transfer. 94 | */ 95 | typedef struct s3_ipc_context 96 | { 97 | char datafn[MAXPGPATH]; 98 | FILE *datafp; 99 | char flagfn[MAXPGPATH]; 100 | FILE *flagfp; 101 | } s3_ipc_context; 102 | 103 | /* 104 | * Function declarations 105 | */ 106 | PG_FUNCTION_INFO_V1(s3test); 107 | PG_FUNCTION_INFO_V1(s3_fdw_handler); 108 | PG_FUNCTION_INFO_V1(s3_fdw_validator); 109 | 110 | Datum s3test(PG_FUNCTION_ARGS); 111 | Datum s3_fdw_handler(PG_FUNCTION_ARGS); 112 | Datum s3_fdw_validator(PG_FUNCTION_ARGS); 113 | 114 | /* 115 | * FDW callback routines 116 | */ 117 | static FdwPlan *s3PlanForeignScan(Oid foreigntableid, 118 | PlannerInfo *root, 119 | RelOptInfo *baserel); 120 | static void s3ExplainForeignScan(ForeignScanState *node, ExplainState *es); 121 | static void s3BeginForeignScan(ForeignScanState *node, int eflags); 122 | static TupleTableSlot *s3IterateForeignScan(ForeignScanState *node); 123 | static void s3ReScanForeignScan(ForeignScanState *node); 124 | static void s3EndForeignScan(ForeignScanState *node); 125 | 126 | /* 127 | * Helper functions 128 | */ 129 | static bool is_valid_option(const char *option, Oid context); 130 | static void s3GetOptions(Oid foreigntableid, S3FdwExecutionState *state); 131 | static void estimate_costs(PlannerInfo *root, RelOptInfo *baserel, 132 | const char *filename, 133 | Cost *startup_cost, Cost *total_cost); 134 | 135 | static size_t header_handler(void *buffer, size_t size, size_t nmemb, void *buf); 136 | static size_t body_handler(void *buffer, size_t size, size_t nmemb, void *userp); 137 | static size_t write_data_to_buf(void *buffer, size_t size, size_t nmemb, void *buf); 138 | 139 | static char *create_tempprefix(char *seed); 140 | //static void s3_resource_release(ResourceReleasePhase phase, bool isCommit, bool isTopLevel, void *arg); 141 | static void s3_on_exit(int code, Datum arg); 142 | 143 | void _PG_init(void); 144 | 145 | Datum 146 | s3test(PG_FUNCTION_ARGS) 147 | { 148 | CURL *curl; 149 | StringInfoData buf; 150 | int sc; 151 | char *url; 152 | char tmp[1024]; 153 | struct curl_slist *slist; 154 | char *datestring; 155 | char *signature; 156 | 157 | char *bucket = "umitanuki-dbtest"; 158 | char *file = "1.txt"; 159 | char *host = "s3-ap-northeast-1.amazonaws.com"; 160 | 161 | char *accesskey = ""; 162 | char *secretkey = ""; 163 | 164 | url = text_to_cstring(PG_GETARG_TEXT_P(0)); 165 | 166 | url = palloc0(1024); 167 | snprintf(url, 1024, "http://%s/%s/%s", host, bucket, file); 168 | datestring = httpdate(NULL); 169 | signature = s3_signature("GET", datestring, bucket, file, secretkey); 170 | 171 | slist = NULL; 172 | snprintf(tmp, sizeof(tmp), "Date: %s", datestring); 173 | slist = curl_slist_append(slist, tmp); 174 | snprintf(tmp, sizeof(tmp), "Authorization: AWS %s:%s", accesskey, signature); 175 | slist = curl_slist_append(slist, tmp); 176 | initStringInfo(&buf); 177 | 178 | curl = curl_easy_init(); 179 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); 180 | curl_easy_setopt(curl, CURLOPT_URL, url); 181 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data_to_buf); 182 | curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buf); 183 | 184 | sc = curl_easy_perform(curl); 185 | 186 | curl_easy_cleanup(curl); 187 | 188 | PG_RETURN_TEXT_P(cstring_to_text(buf.data)); 189 | } 190 | 191 | /* 192 | * Foreign-data wrapper handler function: return a struct with pointers 193 | * to my callback routines. 194 | */ 195 | Datum 196 | s3_fdw_handler(PG_FUNCTION_ARGS) 197 | { 198 | FdwRoutine *fdwroutine = makeNode(FdwRoutine); 199 | 200 | fdwroutine->PlanForeignScan = s3PlanForeignScan; 201 | fdwroutine->ExplainForeignScan = s3ExplainForeignScan; 202 | fdwroutine->BeginForeignScan = s3BeginForeignScan; 203 | fdwroutine->IterateForeignScan = s3IterateForeignScan; 204 | fdwroutine->ReScanForeignScan = s3ReScanForeignScan; 205 | fdwroutine->EndForeignScan = s3EndForeignScan; 206 | 207 | PG_RETURN_POINTER(fdwroutine); 208 | } 209 | 210 | /* 211 | * Validate the generic options given to a FOREIGN DATA WRAPPER, SERVER, 212 | * USER MAPPING or FOREIGN TABLE that uses file_fdw. 213 | * 214 | * Raise an ERROR if the option or its value is considered invalid. 215 | */ 216 | Datum 217 | s3_fdw_validator(PG_FUNCTION_ARGS) 218 | { 219 | List *options_list = untransformRelOptions(PG_GETARG_DATUM(0)); 220 | Oid catalog = PG_GETARG_OID(1); 221 | char *filename = NULL, 222 | *bucketname = NULL, 223 | *hostname = NULL, 224 | *accesskey = NULL, 225 | *secretkey = NULL; 226 | List *copy_options = NIL; 227 | ListCell *cell; 228 | 229 | /* 230 | * Check that only options supported by s3_fdw, and allowed for the 231 | * current object type, are given. 232 | */ 233 | foreach(cell, options_list) 234 | { 235 | DefElem *def = (DefElem *) lfirst(cell); 236 | 237 | if (!is_valid_option(def->defname, catalog)) 238 | { 239 | struct S3FdwOption *opt; 240 | StringInfoData buf; 241 | 242 | /* 243 | * Unknown option specified, complain about it. Provide a hint 244 | * with list of valid options for the object. 245 | */ 246 | initStringInfo(&buf); 247 | for (opt = valid_options; opt->optname; opt++) 248 | { 249 | if (catalog == opt->optcontext) 250 | appendStringInfo(&buf, "%s%s", (buf.len > 0) ? ", " : "", 251 | opt->optname); 252 | } 253 | 254 | ereport(ERROR, 255 | (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), 256 | errmsg("invalid option \"%s\"", def->defname), 257 | errhint("Valid options in this context are: %s", 258 | buf.data))); 259 | } 260 | 261 | /* Separate out filename, since ProcessCopyOptions won't allow it */ 262 | if (strcmp(def->defname, "filename") == 0) 263 | { 264 | if (filename) 265 | ereport(ERROR, 266 | (errcode(ERRCODE_SYNTAX_ERROR), 267 | errmsg("conflicting or redundant options"))); 268 | filename = defGetString(def); 269 | } 270 | else if(strcmp(def->defname, "bucketname") == 0) 271 | { 272 | if (bucketname) 273 | ereport(ERROR, 274 | (errcode(ERRCODE_SYNTAX_ERROR), 275 | errmsg("conflicting or redundant options"))); 276 | bucketname = defGetString(def); 277 | } 278 | else if(strcmp(def->defname, "hostname") == 0) 279 | { 280 | if (hostname) 281 | ereport(ERROR, 282 | (errcode(ERRCODE_SYNTAX_ERROR), 283 | errmsg("conflicting or redundant options"))); 284 | hostname = defGetString(def); 285 | } 286 | else if(strcmp(def->defname, "accesskey") == 0) 287 | { 288 | if (accesskey) 289 | ereport(ERROR, 290 | (errcode(ERRCODE_SYNTAX_ERROR), 291 | errmsg("conflicting or redundant options"))); 292 | accesskey = defGetString(def); 293 | } 294 | else if(strcmp(def->defname, "secretkey") == 0) 295 | { 296 | if (secretkey) 297 | ereport(ERROR, 298 | (errcode(ERRCODE_SYNTAX_ERROR), 299 | errmsg("conflicting or redundant options"))); 300 | secretkey = defGetString(def); 301 | } 302 | else 303 | copy_options = lappend(copy_options, def); 304 | } 305 | 306 | /* 307 | * Now apply the core COPY code's validation logic for more checks. 308 | */ 309 | ProcessCopyOptions(NULL, true, copy_options); 310 | 311 | /* 312 | * Hostname option is required for s3_fdw foreign tables. 313 | */ 314 | if (catalog == ForeignTableRelationId && hostname == NULL) 315 | ereport(ERROR, 316 | (errcode(ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED), 317 | errmsg("hostname is required for s3_fdw foreign tables"))); 318 | 319 | /* 320 | * Bucketname option is required for s3_fdw foreign tables. 321 | */ 322 | if (catalog == ForeignTableRelationId && bucketname == NULL) 323 | ereport(ERROR, 324 | (errcode(ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED), 325 | errmsg("bucketname is required for s3_fdw foreign tables"))); 326 | 327 | /* 328 | * Filename option is required for s3_fdw foreign tables. 329 | */ 330 | if (catalog == ForeignTableRelationId && filename == NULL) 331 | ereport(ERROR, 332 | (errcode(ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED), 333 | errmsg("filename is required for s3_fdw foreign tables"))); 334 | 335 | PG_RETURN_VOID(); 336 | } 337 | 338 | /* 339 | * Check if the provided option is one of the valid options. 340 | * context is the Oid of the catalog holding the object the option is for. 341 | */ 342 | static bool 343 | is_valid_option(const char *option, Oid context) 344 | { 345 | struct S3FdwOption *opt; 346 | 347 | for (opt = valid_options; opt->optname; opt++) 348 | { 349 | if (context == opt->optcontext && strcmp(opt->optname, option) == 0) 350 | return true; 351 | } 352 | return false; 353 | } 354 | 355 | /* 356 | * Fetch the options for a s3_fdw foreign table. 357 | * 358 | * We have to separate out "filename", "bucketname" and "hostname" 359 | * from the other options because it must not appear in the options 360 | * list passed to the core COPY code. 361 | */ 362 | static void 363 | s3GetOptions(Oid foreigntableid, S3FdwExecutionState *state) 364 | { 365 | ForeignTable *table; 366 | ForeignServer *server; 367 | ForeignDataWrapper *wrapper; 368 | UserMapping *mapping; 369 | List *options, *new_options; 370 | ListCell *lc; 371 | 372 | /* 373 | * Extract options from FDW objects. 374 | */ 375 | table = GetForeignTable(foreigntableid); 376 | server = GetForeignServer(table->serverid); 377 | wrapper = GetForeignDataWrapper(server->fdwid); 378 | mapping = GetUserMapping(GetUserId(), table->serverid); 379 | 380 | options = NIL; 381 | options = list_concat(options, wrapper->options); 382 | options = list_concat(options, server->options); 383 | options = list_concat(options, mapping->options); 384 | options = list_concat(options, table->options); 385 | 386 | /* 387 | * Separate out the host, bucket and filename. 388 | */ 389 | state->hostname = NULL; 390 | state->bucketname = NULL; 391 | state->filename = NULL; 392 | new_options = NIL; 393 | foreach(lc, options) 394 | { 395 | DefElem *def = (DefElem *) lfirst(lc); 396 | 397 | if (strcmp(def->defname, "hostname") == 0) 398 | { 399 | state->hostname = defGetString(def); 400 | } 401 | else if (strcmp(def->defname, "bucketname") == 0) 402 | { 403 | state->bucketname = defGetString(def); 404 | } 405 | else if (strcmp(def->defname, "filename") == 0) 406 | { 407 | state->filename = defGetString(def); 408 | } 409 | else if (strcmp(def->defname, "accesskey") == 0) 410 | { 411 | state->accesskey = defGetString(def); 412 | } 413 | else if (strcmp(def->defname, "secretkey") == 0) 414 | { 415 | state->secretkey = defGetString(def); 416 | } 417 | else 418 | new_options = lappend(new_options, def); 419 | } 420 | 421 | /* 422 | * The validator should have checked those mandatory options were 423 | * included in the options, but check again, just in case. 424 | */ 425 | if (state->hostname == NULL) 426 | elog(ERROR, "hostname is required for s3_fdw foreign tables"); 427 | if (state->bucketname == NULL) 428 | elog(ERROR, "bucketname is required for s3_fdw foreign tables"); 429 | if (state->filename == NULL) 430 | elog(ERROR, "filename is required for s3_fdw foreign tables"); 431 | 432 | state->copy_options = new_options; 433 | } 434 | 435 | /* 436 | * s3PlanForeignScan 437 | * Create a FdwPlan for a scan on the foreign table 438 | */ 439 | static FdwPlan * 440 | s3PlanForeignScan(Oid foreigntableid, 441 | PlannerInfo *root, 442 | RelOptInfo *baserel) 443 | { 444 | FdwPlan *fdwplan; 445 | S3FdwExecutionState state; 446 | 447 | /* Fetch options -- it's not sure what is needed here */ 448 | s3GetOptions(foreigntableid, &state); 449 | 450 | /* Construct FdwPlan with cost estimates */ 451 | fdwplan = makeNode(FdwPlan); 452 | estimate_costs(root, baserel, state.filename, 453 | &fdwplan->startup_cost, &fdwplan->total_cost); 454 | fdwplan->fdw_private = NIL; /* not used */ 455 | 456 | return fdwplan; 457 | } 458 | 459 | /* 460 | * s3ExplainForeignScan 461 | * Produce extra output for EXPLAIN 462 | */ 463 | static void 464 | s3ExplainForeignScan(ForeignScanState *node, ExplainState *es) 465 | { 466 | S3FdwExecutionState state; 467 | StringInfoData url; 468 | 469 | initStringInfo(&url); 470 | 471 | /* Fetch options */ 472 | s3GetOptions(RelationGetRelid(node->ss.ss_currentRelation), &state); 473 | appendStringInfo(&url, "http://%s/%s/%s", 474 | state.hostname, state.bucketname, state.filename); 475 | 476 | ExplainPropertyText("Foreign URL", url.data, es); 477 | } 478 | 479 | /* 480 | * s3BeginForeignScan 481 | * Initiate access to the file by creating CopyState 482 | */ 483 | static void 484 | s3BeginForeignScan(ForeignScanState *node, int eflags) 485 | { 486 | CopyState cstate; 487 | S3FdwExecutionState *festate; 488 | StringInfoData buf; 489 | char *url, *datestring, *signature; 490 | char *prefix; 491 | pid_t pid; 492 | s3_ipc_context ctx; 493 | 494 | /* 495 | * Do nothing in EXPLAIN (no ANALYZE) case. node->fdw_state stays NULL. 496 | */ 497 | if (eflags & EXEC_FLAG_EXPLAIN_ONLY) 498 | return; 499 | 500 | festate = (S3FdwExecutionState *) palloc(sizeof(S3FdwExecutionState)); 501 | /* Fetch options of foreign table */ 502 | s3GetOptions(RelationGetRelid(node->ss.ss_currentRelation), festate); 503 | 504 | initStringInfo(&buf); 505 | appendStringInfo(&buf, "http://%s/%s/%s", 506 | festate->hostname, festate->bucketname, festate->filename); 507 | url = pstrdup(buf.data); 508 | 509 | datestring = httpdate(NULL); 510 | signature = s3_signature("GET", datestring, 511 | festate->bucketname, festate->filename, festate->secretkey); 512 | 513 | prefix = create_tempprefix(signature); 514 | snprintf(ctx.flagfn, sizeof(ctx.flagfn), "%s.flag", prefix); 515 | snprintf(ctx.datafn, sizeof(ctx.datafn), "%s.data", prefix); 516 | // unlink(ctx.flagfn); 517 | // unlink(ctx.datafn); 518 | if (mkfifo(ctx.flagfn, S_IRUSR | S_IWUSR) != 0) 519 | elog(ERROR, "mkfifo failed(%d):%s", errno, ctx.flagfn); 520 | if (mkfifo(ctx.datafn, S_IRUSR | S_IWUSR) != 0) 521 | elog(ERROR, "mkfifo failed(%d):%s", errno, ctx.datafn); 522 | 523 | 524 | /* 525 | * Fork to maximize parallelism of input from HTTP and output to SQL. 526 | * The spawned child process cheats by on_exit_rest() to die immediately. 527 | */ 528 | pid = fork_process(); 529 | if (pid == 0) /* child */ 530 | { 531 | struct curl_slist *slist; 532 | CURL *curl; 533 | int sc; 534 | 535 | MyProcPid = getpid(); /* reset MyProcPid */ 536 | 537 | /* 538 | * The exit callback routines clean up 539 | * unnecessary resources holded the parent process. 540 | * The child dies silently when finishing its job. 541 | */ 542 | on_exit_reset(); 543 | 544 | /* 545 | * Set up request header list 546 | */ 547 | slist = NULL; 548 | initStringInfo(&buf); 549 | appendStringInfo(&buf, "Date: %s", datestring); 550 | slist = curl_slist_append(slist, buf.data); 551 | resetStringInfo(&buf); 552 | appendStringInfo(&buf, "Authorization: AWS %s:%s", 553 | festate->accesskey, signature); 554 | slist = curl_slist_append(slist, buf.data); 555 | /* 556 | * Set up CURL instance. 557 | */ 558 | curl = curl_easy_init(); 559 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, slist); 560 | curl_easy_setopt(curl, CURLOPT_URL, url); 561 | curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, header_handler); 562 | curl_easy_setopt(curl, CURLOPT_HEADERDATA, &ctx); 563 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, body_handler); 564 | curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ctx); 565 | sc = curl_easy_perform(curl); 566 | if (ctx.datafp) 567 | FreeFile(ctx.datafp); 568 | if (sc != 0) 569 | { 570 | elog(NOTICE, "%s:curl_easy_perform = %d", url, sc); 571 | unlink(ctx.datafn); 572 | } 573 | curl_slist_free_all(slist); 574 | curl_easy_cleanup(curl); 575 | 576 | proc_exit(0); 577 | } 578 | elog(DEBUG1, "child pid = %d", pid); 579 | 580 | { 581 | int status; 582 | FILE *fp; 583 | 584 | fp = AllocateFile(ctx.flagfn, PG_BINARY_R); 585 | read(fileno(fp), &status, sizeof(int)); 586 | FreeFile(fp); 587 | unlink(ctx.flagfn); 588 | if (status != 200) 589 | { 590 | elog(ERROR, "bad input from API. Status code: %d", status); 591 | } 592 | } 593 | 594 | /* 595 | * Create CopyState from FDW options. We always acquire all columns, so 596 | * as to match the expected ScanTupleSlot signature. 597 | */ 598 | cstate = BeginCopyFrom(node->ss.ss_currentRelation, 599 | ctx.datafn, 600 | NIL, 601 | festate->copy_options); 602 | 603 | /* 604 | * Save state in node->fdw_state. We must save enough information to call 605 | * BeginCopyFrom() again. 606 | */ 607 | festate->cstate = cstate; 608 | festate->datafn = pstrdup(ctx.datafn); 609 | 610 | node->fdw_state = (void *) festate; 611 | } 612 | 613 | /* 614 | * s3IterateForeignScan 615 | * Read next record from the data file and store it into the 616 | * ScanTupleSlot as a virtual tuple 617 | */ 618 | static TupleTableSlot * 619 | s3IterateForeignScan(ForeignScanState *node) 620 | { 621 | S3FdwExecutionState *festate = (S3FdwExecutionState *) node->fdw_state; 622 | TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; 623 | bool found; 624 | ErrorContextCallback errcontext; 625 | 626 | /* Set up callback to identify error line number. */ 627 | errcontext.callback = CopyFromErrorCallback; 628 | errcontext.arg = (void *) festate->cstate; 629 | errcontext.previous = error_context_stack; 630 | error_context_stack = &errcontext; 631 | 632 | /* 633 | * The protocol for loading a virtual tuple into a slot is first 634 | * ExecClearTuple, then fill the values/isnull arrays, then 635 | * ExecStoreVirtualTuple. If we don't find another row in the file, we 636 | * just skip the last step, leaving the slot empty as required. 637 | * 638 | * We can pass ExprContext = NULL because we read all columns from the 639 | * file, so no need to evaluate default expressions. 640 | * 641 | * We can also pass tupleOid = NULL because we don't allow oids for 642 | * foreign tables. 643 | */ 644 | ExecClearTuple(slot); 645 | found = NextCopyFrom(festate->cstate, NULL, 646 | slot->tts_values, slot->tts_isnull, 647 | NULL); 648 | if (found) 649 | ExecStoreVirtualTuple(slot); 650 | 651 | /* Remove error callback. */ 652 | error_context_stack = errcontext.previous; 653 | 654 | return slot; 655 | } 656 | 657 | /* 658 | * s3EndForeignScan 659 | * Finish scanning foreign table and dispose objects used for this scan 660 | */ 661 | static void 662 | s3EndForeignScan(ForeignScanState *node) 663 | { 664 | S3FdwExecutionState *festate = (S3FdwExecutionState *) node->fdw_state; 665 | 666 | /* if festate is NULL, we are in EXPLAIN; nothing to do */ 667 | if (festate) 668 | { 669 | EndCopyFrom(festate->cstate); 670 | unlink(festate->datafn); 671 | } 672 | } 673 | 674 | /* 675 | * s3ReScanForeignScan 676 | * Rescan table, possibly with new parameters 677 | */ 678 | static void 679 | s3ReScanForeignScan(ForeignScanState *node) 680 | { 681 | S3FdwExecutionState *festate = (S3FdwExecutionState *) node->fdw_state; 682 | 683 | EndCopyFrom(festate->cstate); 684 | 685 | festate->cstate = BeginCopyFrom(node->ss.ss_currentRelation, 686 | festate->filename, 687 | NIL, 688 | festate->copy_options); 689 | } 690 | 691 | /* 692 | * Estimate costs of scanning a foreign table. 693 | */ 694 | static void 695 | estimate_costs(PlannerInfo *root, RelOptInfo *baserel, 696 | const char *filename, 697 | Cost *startup_cost, Cost *total_cost) 698 | { 699 | // struct stat stat_buf; 700 | BlockNumber pages; 701 | // int tuple_width; 702 | double ntuples; 703 | double nrows; 704 | Cost run_cost = 0; 705 | Cost cpu_per_tuple; 706 | 707 | /* 708 | * Get size of the file. It might not be there at plan time, though, in 709 | * which case we have to use a default estimate. 710 | */ 711 | // if (stat(filename, &stat_buf) < 0) 712 | // stat_buf.st_size = 10 * BLCKSZ; 713 | 714 | /* 715 | * Convert size to pages for use in I/O cost estimate below. 716 | */ 717 | // pages = (stat_buf.st_size + (BLCKSZ - 1)) / BLCKSZ; 718 | // if (pages < 1) 719 | // pages = 1; 720 | pages = 10; 721 | 722 | /* 723 | * Estimate the number of tuples in the file. We back into this estimate 724 | * using the planner's idea of the relation width; which is bogus if not 725 | * all columns are being read, not to mention that the text representation 726 | * of a row probably isn't the same size as its internal representation. 727 | * FIXME later. 728 | */ 729 | // tuple_width = MAXALIGN(baserel->width) + MAXALIGN(sizeof(HeapTupleHeaderData)); 730 | 731 | // ntuples = clamp_row_est((double) stat_buf.st_size / (double) tuple_width); 732 | ntuples = 1000; 733 | 734 | /* 735 | * Now estimate the number of rows returned by the scan after applying the 736 | * baserestrictinfo quals. This is pretty bogus too, since the planner 737 | * will have no stats about the relation, but it's better than nothing. 738 | */ 739 | nrows = ntuples * 740 | clauselist_selectivity(root, 741 | baserel->baserestrictinfo, 742 | 0, 743 | JOIN_INNER, 744 | NULL); 745 | 746 | nrows = clamp_row_est(nrows); 747 | 748 | /* Save the output-rows estimate for the planner */ 749 | baserel->rows = nrows; 750 | 751 | /* 752 | * Now estimate costs. We estimate costs almost the same way as 753 | * cost_seqscan(), thus assuming that I/O costs are equivalent to a 754 | * regular table file of the same size. However, we take per-tuple CPU 755 | * costs as 10x of a seqscan, to account for the cost of parsing records. 756 | */ 757 | run_cost += seq_page_cost * pages; 758 | 759 | *startup_cost = baserel->baserestrictcost.startup; 760 | cpu_per_tuple = cpu_tuple_cost * 10 + baserel->baserestrictcost.per_tuple; 761 | run_cost += cpu_per_tuple * ntuples; 762 | *total_cost = *startup_cost + run_cost; 763 | } 764 | 765 | static size_t 766 | header_handler(void *buffer, size_t size, size_t nmemb, void *userp) 767 | { 768 | const char *HTTP_1_1 = "HTTP/1.1"; 769 | size_t segsize = size * nmemb; 770 | s3_ipc_context *ctx = (s3_ipc_context *) userp; 771 | 772 | if (strncmp(buffer, HTTP_1_1, strlen(HTTP_1_1)) == 0) 773 | { 774 | int status; 775 | 776 | status = atoi((char *) buffer + strlen(HTTP_1_1) + 1); 777 | ctx->flagfp = AllocateFile(ctx->flagfn, PG_BINARY_W); 778 | write(fileno(ctx->flagfp), &status, sizeof(int)); 779 | FreeFile(ctx->flagfp); 780 | if (status != 200) 781 | { 782 | ctx->datafp = NULL; 783 | /* interrupt */ 784 | return 0; 785 | } 786 | /* iif success */ 787 | // ctx->datafp = AllocateFile(ctx->dataname, PG_BINARY_W); 788 | ctx->datafp = AllocateFile(ctx->datafn, PG_BINARY_W); 789 | } 790 | 791 | return segsize; 792 | } 793 | 794 | static size_t 795 | body_handler(void *buffer, size_t size, size_t nmemb, void *userp) 796 | { 797 | size_t segsize = size * nmemb; 798 | s3_ipc_context *ctx = (s3_ipc_context *) userp; 799 | 800 | fwrite(buffer, size, nmemb, ctx->datafp); 801 | 802 | return segsize; 803 | } 804 | 805 | static size_t 806 | write_data_to_buf(void *buffer, size_t size, size_t nmemb, void *userp) 807 | { 808 | size_t segsize = size * nmemb; 809 | StringInfo info = (StringInfo) userp; 810 | 811 | appendBinaryStringInfo(info, (const char *) buffer, segsize); 812 | 813 | return segsize; 814 | } 815 | 816 | static char * 817 | create_tempprefix(char *seed) 818 | { 819 | char filename[MAXPGPATH], path[MAXPGPATH], *s; 820 | 821 | snprintf(filename, sizeof(filename), "%u.%s", MyProcPid, seed); 822 | s = &filename[0]; 823 | while(*s) 824 | { 825 | if (*s == '/') 826 | *s = '%'; 827 | s++; 828 | } 829 | mkdir("base/" PG_TEMP_FILES_DIR, S_IRWXU); 830 | snprintf(path, sizeof(path), "base/%s/%s", PG_TEMP_FILES_DIR, filename); 831 | 832 | return pstrdup(path); 833 | } 834 | 835 | static bool 836 | presuffix_test(char *base, char *prefix, char *suffix) 837 | { 838 | int len, plen, slen; 839 | 840 | len = strlen(base); 841 | plen = strlen(prefix); 842 | slen = strlen(suffix); 843 | if (len < plen + slen) 844 | return false; 845 | 846 | return memcmp(base, prefix, plen) == 0 && 847 | memcmp(base + len - slen, suffix, slen) == 0; 848 | } 849 | 850 | /* 851 | * Clean up fifos on process exit. 852 | * We don't care other process's fifo since it may be 853 | * in use right now. It might be better to delete fifos 854 | * as soone as possible, but they don't consume disk space 855 | * so let's postpone it till exit, where it's sure to delete. 856 | */ 857 | static void 858 | s3_on_exit(int code, Datum arg) 859 | { 860 | char prefix[32]; 861 | char *dirname = "base/" PG_TEMP_FILES_DIR; 862 | DIR *dir; 863 | struct dirent *ent; 864 | 865 | snprintf(prefix, sizeof(prefix), "%d.", MyProcPid); 866 | dir = AllocateDir(dirname); 867 | while((ent = ReadDir(dir, dirname)) != NULL) 868 | { 869 | int len; 870 | 871 | len = strlen(ent->d_name); 872 | if (presuffix_test(ent->d_name, prefix, ".data") || 873 | presuffix_test(ent->d_name, prefix, ".flag")) 874 | { 875 | unlink(ent->d_name); 876 | } 877 | } 878 | 879 | FreeDir(dir); 880 | } 881 | 882 | void 883 | _PG_init(void) 884 | { 885 | on_proc_exit(s3_on_exit, (Datum) 0); 886 | // RegisterResourceReleaseCallback(s3_resource_release, NULL); 887 | } 888 | -------------------------------------------------------------------------------- /s3_fdw.control: -------------------------------------------------------------------------------- 1 | # s3fdw extension 2 | comment = 'foreign-data wrapper for Amazon S3' 3 | default_version = '0.1.0' 4 | module_pathname = '$libdir/s3_fdw' 5 | relocatable = true 6 | -------------------------------------------------------------------------------- /s3_fdw.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION s3test(text) RETURNS text AS 2 | 'MODULE_PATHNAME' LANGUAGE C; 3 | 4 | CREATE FUNCTION s3_fdw_handler() RETURNS fdw_handler AS 5 | 'MODULE_PATHNAME' LANGUAGE C; 6 | CREATE FUNCTION s3_fdw_validator(text[], oid) RETURNS void AS 7 | 'MODULE_PATHNAME' LANGUAGE C; 8 | 9 | CREATE FOREIGN DATA WRAPPER s3_fdw 10 | HANDLER s3_fdw_handler 11 | VALIDATOR s3_fdw_validator; 12 | --------------------------------------------------------------------------------