├── .gitignore ├── HISTORY.md ├── tests ├── redis.commands ├── expected_outputs │ ├── 10 │ ├── 11 │ ├── 12 │ ├── 9.4 │ ├── 9.5 │ └── 9.6 ├── helpers.sh ├── setup.sql ├── run.sql ├── setups │ ├── 10.sql │ ├── 11.sql │ ├── 12.sql │ ├── 9.5.sql │ ├── 9.6.sql │ └── 9.4.sql ├── Dockerfiles │ ├── 10 │ ├── 11 │ ├── 12 │ ├── 9.5 │ ├── 9.6 │ └── 9.4 └── run.sh ├── .dockerignore ├── scripts ├── run_tests ├── build_image └── deploy ├── LICENSE.md ├── holycorn.control ├── plan_state.h ├── builtin_wrappers └── holycorn-redis │ ├── mrbgem.rake │ ├── src │ ├── holycorn_redis.h │ └── holycorn_redis.c │ └── mrblib │ └── holycorn-redis.rb ├── execution_state.h ├── examples ├── simple_strings.rb ├── redis.rb └── openweathermap.rb ├── options.h ├── Changelog ├── holycorn--1.0.sql ├── Makefile ├── Dockerfile ├── Rakefile ├── META.json ├── config └── build_config.rb ├── .circleci └── config.yml ├── README.md └── holycorn.c /.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | vendor/* 4 | *.rdb 5 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | # HISTORY 2 | 3 | ## 1.0.0 4 | 5 | * First alpha release 6 | -------------------------------------------------------------------------------- /tests/redis.commands: -------------------------------------------------------------------------------- 1 | flushall 2 | set foo 1 3 | set bar 2 4 | set baz 3 5 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.so 3 | vendor/* 4 | *.rdb 5 | .git 6 | Dockerfile 7 | tests/Dockerfiles* 8 | -------------------------------------------------------------------------------- /scripts/run_tests: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | docker run franckverrot/holycorn tests/run.sh $PG_VERSION 3 | -------------------------------------------------------------------------------- /tests/expected_outputs/10: -------------------------------------------------------------------------------- 1 | server_version_num,100 2 | key,bar;value,2;;key,baz;value,3;;key,foo;value,1 3 | -------------------------------------------------------------------------------- /tests/expected_outputs/11: -------------------------------------------------------------------------------- 1 | server_version_num,110 2 | key,bar;value,2;;key,baz;value,3;;key,foo;value,1 3 | -------------------------------------------------------------------------------- /tests/expected_outputs/12: -------------------------------------------------------------------------------- 1 | server_version_num,120 2 | key,bar;value,2;;key,baz;value,3;;key,foo;value,1 3 | -------------------------------------------------------------------------------- /tests/expected_outputs/9.4: -------------------------------------------------------------------------------- 1 | server_version_num,904 2 | key,bar;value,2;;key,baz;value,3;;key,foo;value,1 3 | -------------------------------------------------------------------------------- /tests/expected_outputs/9.5: -------------------------------------------------------------------------------- 1 | server_version_num,905 2 | key,bar;value,2;;key,baz;value,3;;key,foo;value,1 3 | -------------------------------------------------------------------------------- /tests/expected_outputs/9.6: -------------------------------------------------------------------------------- 1 | server_version_num,906 2 | key,bar;value,2;;key,baz;value,3;;key,foo;value,1 3 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Please see [README.md](https://github.com/franckverrot/holycorn#license) for licensing details. 2 | -------------------------------------------------------------------------------- /tests/helpers.sh: -------------------------------------------------------------------------------- 1 | PSQL="psql -p 5433" 2 | function exec_psql { 3 | su -c "$PSQL -q < $1" - postgres 4 | } 5 | 6 | -------------------------------------------------------------------------------- /scripts/build_image: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | docker build -f tests/Dockerfiles/$PG_VERSION -t franckverrot/holycorn . 3 | -------------------------------------------------------------------------------- /scripts/deploy: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -ex 3 | sudo pgxn install pgxn_utils 4 | pgxn-utils bundle 5 | pgxn-utils release ../holycorn*zip 6 | -------------------------------------------------------------------------------- /tests/setup.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION holycorn; 2 | CREATE SCHEMA holycorn_tables; 3 | CREATE SERVER holycorn_server 4 | FOREIGN DATA WRAPPER holycorn; 5 | -------------------------------------------------------------------------------- /holycorn.control: -------------------------------------------------------------------------------- 1 | # holycorn extension 2 | comment = 'Ruby foreign-data wrapper provider' 3 | default_version = '1.0' 4 | module_pathname = '$libdir/holycorn' 5 | relocatable = true 6 | -------------------------------------------------------------------------------- /tests/run.sql: -------------------------------------------------------------------------------- 1 | SELECT left(current_setting('server_version_num'), 3) as server_version_num; 2 | SELECT 3 | key 4 | , value 5 | FROM 6 | holycorn_tables.holycorn_redis_table 7 | ORDER BY 8 | key 9 | -------------------------------------------------------------------------------- /plan_state.h: -------------------------------------------------------------------------------- 1 | typedef struct HolycornPlanState { 2 | char *wrapper_path; 3 | char *wrapper_class; 4 | List *options; 5 | BlockNumber pages; 6 | double ntuples; 7 | } HolycornPlanState; 8 | -------------------------------------------------------------------------------- /builtin_wrappers/holycorn-redis/mrbgem.rake: -------------------------------------------------------------------------------- 1 | MRuby::Gem::Specification.new('holycorn-redis') do |spec| 2 | spec.license = 'LGPLv3' 3 | spec.author = 'Franck Verrot' 4 | spec.summary = 'Holycorn Redis Foreign Data Wrapper' 5 | end 6 | -------------------------------------------------------------------------------- /execution_state.h: -------------------------------------------------------------------------------- 1 | typedef struct HolycornExecutionState 2 | { 3 | char *wrapper_path; 4 | mrb_value * mrb_state; 5 | mrb_value iterator; 6 | List *options; 7 | mrb_value options_hash; 8 | int passes; 9 | } HolycornExecutionState; 10 | -------------------------------------------------------------------------------- /builtin_wrappers/holycorn-redis/src/holycorn_redis.h: -------------------------------------------------------------------------------- 1 | #ifndef HOLYCORN_REDIS_H 2 | #define HOLYCORN_REDIS_H 3 | 4 | #include "mruby.h" 5 | void mrb_holycorn_redis_gem_init(mrb_state *mrb); 6 | void mrb_holycorn_redis_gem_final(mrb_state *mrb); 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /tests/setups/10.sql: -------------------------------------------------------------------------------- 1 | IMPORT FOREIGN SCHEMA holycorn_schema 2 | FROM SERVER holycorn_server 3 | INTO holycorn_tables 4 | OPTIONS ( wrapper_class 'HolycornRedis' 5 | , host '127.0.0.1' 6 | , port '6379' 7 | , db '0' 8 | , prefix 'holycorn_' 9 | ); 10 | -------------------------------------------------------------------------------- /tests/setups/11.sql: -------------------------------------------------------------------------------- 1 | IMPORT FOREIGN SCHEMA holycorn_schema 2 | FROM SERVER holycorn_server 3 | INTO holycorn_tables 4 | OPTIONS ( wrapper_class 'HolycornRedis' 5 | , host '127.0.0.1' 6 | , port '6379' 7 | , db '0' 8 | , prefix 'holycorn_' 9 | ); 10 | -------------------------------------------------------------------------------- /tests/setups/12.sql: -------------------------------------------------------------------------------- 1 | IMPORT FOREIGN SCHEMA holycorn_schema 2 | FROM SERVER holycorn_server 3 | INTO holycorn_tables 4 | OPTIONS ( wrapper_class 'HolycornRedis' 5 | , host '127.0.0.1' 6 | , port '6379' 7 | , db '0' 8 | , prefix 'holycorn_' 9 | ); 10 | -------------------------------------------------------------------------------- /tests/setups/9.5.sql: -------------------------------------------------------------------------------- 1 | IMPORT FOREIGN SCHEMA holycorn_schema 2 | FROM SERVER holycorn_server 3 | INTO holycorn_tables 4 | OPTIONS ( wrapper_class 'HolycornRedis' 5 | , host '127.0.0.1' 6 | , port '6379' 7 | , db '0' 8 | , prefix 'holycorn_' 9 | ); 10 | -------------------------------------------------------------------------------- /tests/setups/9.6.sql: -------------------------------------------------------------------------------- 1 | IMPORT FOREIGN SCHEMA holycorn_schema 2 | FROM SERVER holycorn_server 3 | INTO holycorn_tables 4 | OPTIONS ( wrapper_class 'HolycornRedis' 5 | , host '127.0.0.1' 6 | , port '6379' 7 | , db '0' 8 | , prefix 'holycorn_' 9 | ); 10 | -------------------------------------------------------------------------------- /tests/setups/9.4.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE holycorn_tables.holycorn_redis_table 2 | ( key text 3 | , value text 4 | ) 5 | SERVER holycorn_server 6 | OPTIONS ( wrapper_class 'HolycornRedis' 7 | , host '127.0.0.1' 8 | , port '6379' 9 | , db '0' 10 | ); 11 | -------------------------------------------------------------------------------- /examples/simple_strings.rb: -------------------------------------------------------------------------------- 1 | class Producer 2 | def initialize(env = {}) # env contains informations provided by Holycorn 3 | end 4 | 5 | def each 6 | @enum ||= Enumerator.new do |y| 7 | 10.times do |t| 8 | y.yield [ "Hello #{t}" ] 9 | end 10 | end 11 | @enum.next 12 | end 13 | self 14 | end 15 | -------------------------------------------------------------------------------- /builtin_wrappers/holycorn-redis/src/holycorn_redis.c: -------------------------------------------------------------------------------- 1 | #include "holycorn_redis.h" 2 | 3 | void mrb_holycorn_redis_gem_init(mrb_state *mrb) { 4 | struct RClass *holycorn_redis; 5 | 6 | holycorn_redis = mrb_define_class(mrb, "HolycornRedis", mrb->object_class); 7 | } 8 | 9 | void mrb_holycorn_redis_gem_final(mrb_state *mrb) { 10 | } 11 | -------------------------------------------------------------------------------- /examples/redis.rb: -------------------------------------------------------------------------------- 1 | class RedisFDW 2 | def initialize(env = {}) 3 | @count = 0 4 | 5 | @r = Redis.new "127.0.0.1", 6379 6 | @r.select 0 7 | end 8 | 9 | def each 10 | if (val = @r.randomkey) && (@count < 100) 11 | @count += 1 12 | [val] 13 | else 14 | nil 15 | end 16 | end 17 | self 18 | end 19 | -------------------------------------------------------------------------------- /options.h: -------------------------------------------------------------------------------- 1 | struct HolycornOption { 2 | const char *optname; 3 | Oid optcontext; 4 | }; 5 | 6 | // Only allow setting the repository's path 7 | static const struct HolycornOption valid_options[] = { 8 | {"wrapper_path", ForeignTableRelationId, false}, 9 | {"wrapper_class", ForeignTableRelationId, false}, 10 | {NULL, InvalidOid, false} 11 | }; 12 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | # Release 1.2.0 (unreleased) 2 | 3 | * TBD 4 | 5 | 6 | # Release 1.1.0 7 | 8 | * Support builtin FDW for easy configuration 9 | * Start implementing Redis's FDW 10 | * Dropped support for PG 9.1, 9.2 and 9.3 11 | * Added test infrastructure 12 | 13 | 14 | # Release 1.0.0 15 | 16 | * Initial release 17 | * Ruby wrapper classes receives an environment `Hash` in their constructor 18 | -------------------------------------------------------------------------------- /holycorn--1.0.sql: -------------------------------------------------------------------------------- 1 | \echo Use "CREATE EXTENSION holycorn" to load this file. \quit 2 | 3 | CREATE FUNCTION holycorn_handler() 4 | RETURNS fdw_handler 5 | AS 'MODULE_PATHNAME' 6 | LANGUAGE C STRICT; 7 | 8 | CREATE FUNCTION holycorn_validator(text[], oid) 9 | RETURNS void 10 | AS 'MODULE_PATHNAME' 11 | LANGUAGE C STRICT; 12 | 13 | CREATE FOREIGN DATA WRAPPER holycorn 14 | HANDLER holycorn_handler 15 | VALIDATOR holycorn_validator; 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MODULES = holycorn 2 | MODULE_big = holycorn 3 | 4 | PG_CPPFLAGS = -g -Ivendor/mruby/include -lm 5 | EXTENSION = holycorn 6 | SHLIB_LINK = vendor/mruby/build/i686-pc-linux-gnu/lib/libmruby.a vendor/mruby/build/i686-pc-linux-gnu/mrbgems/mruby-redis/hiredis/libhiredis.a 7 | OBJS = holycorn.o 8 | DATA = holycorn--1.0.sql 9 | PGFILEDESC = "holycorn - Ruby foreign data wrapper provider" 10 | 11 | PG_CONFIG = pg_config 12 | PGXS := $(shell $(PG_CONFIG) --pgxs) 13 | include $(PGXS) 14 | -------------------------------------------------------------------------------- /examples/openweathermap.rb: -------------------------------------------------------------------------------- 1 | class OpenWeatherMap 2 | def initialize(env = {}) 3 | @url = "http://api.openweathermap.org/data/2.5/forecast/daily?q=Lyon&mode=json&units=metric&cnt=7" 4 | 5 | http = HttpRequest.new() 6 | body = http.get(@url).body 7 | @results = JSON.parse(body)['list'].to_enum 8 | end 9 | 10 | def each 11 | if record = @results.next 12 | record.values_at('deg','speed').map(&:to_s) 13 | else 14 | nil 15 | end 16 | end 17 | self 18 | end 19 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:latest 2 | MAINTAINER Franck Verrot 3 | 4 | RUN apt-get update -qq && \ 5 | apt-get install -y \ 6 | build-essential \ 7 | flex bison \ 8 | git \ 9 | libc6-dev-i386 \ 10 | libpq-dev \ 11 | postgresql-9.5 \ 12 | postgresql-server-dev-all \ 13 | rake \ 14 | redis-server 15 | 16 | ADD . /holycorn 17 | WORKDIR /holycorn 18 | 19 | RUN pg_createcluster -p 5433 9.5 my_cluster 20 | RUN rake build && make install 21 | -------------------------------------------------------------------------------- /tests/Dockerfiles/9.5: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | MAINTAINER Franck Verrot 3 | 4 | RUN apt-get update -qq && \ 5 | apt-get install -y \ 6 | build-essential \ 7 | flex bison \ 8 | git \ 9 | libc6-dev-i386 \ 10 | libpq-dev \ 11 | postgresql-9.5 \ 12 | postgresql-server-dev-9.5 \ 13 | rake \ 14 | redis-server 15 | 16 | RUN pg_createcluster -p 5433 9.5 my_cluster 17 | 18 | WORKDIR /holycorn 19 | ADD . /holycorn 20 | 21 | RUN rake build && make install 22 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'fileutils' 2 | 3 | MRubyRCompilationFailed = Class.new(RuntimeError) 4 | 5 | desc "Vendor mruby in a vendor directory" 6 | task :vendor_mruby do 7 | FileUtils.mkdir_p('vendor') 8 | chdir('vendor') do 9 | sh "git clone --depth=1 https://github.com/mruby/mruby.git" 10 | end 11 | end 12 | 13 | desc "Build mruby" 14 | task :build_mruby do 15 | chdir('vendor/mruby') do 16 | FileUtils.cp('../../config/build_config.rb', '.') 17 | sh "make" 18 | end 19 | end 20 | 21 | desc "Clean vendor" 22 | task :clean do 23 | FileUtils.rm_rf('vendor') 24 | end 25 | 26 | task :build => [:clean, :vendor_mruby, :build_mruby] 27 | 28 | task :default do 29 | system("./scripts/build_image && ./scripts/run_tests") 30 | exit $?.exitstatus 31 | end 32 | -------------------------------------------------------------------------------- /tests/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | source $(dirname $0)/helpers.sh 3 | 4 | # Create and start services 5 | pg_ctlcluster $1 my_cluster start 6 | redis-server --daemonize yes 7 | 8 | # Setup Redis 9 | redis-cli < tests/redis.commands 10 | 11 | # Setup Postgres 12 | exec_psql /holycorn/tests/setup.sql 13 | exec_psql /holycorn/tests/setups/$1.sql 14 | exec_psql /holycorn/tests/run.sql 15 | 16 | # Execute tests 17 | expected_output=$(cat tests/expected_outputs/$1) 18 | actual_output=$(su -c "$PSQL -txqAF, -R\; < /holycorn/tests/run.sql" - postgres) 19 | 20 | # Assert results 21 | if [ "$expected_output" == "$actual_output" ] 22 | then 23 | echo "OK" 24 | exit 0 25 | else 26 | echo "KO, expected:" 27 | echo "$expected_output" 28 | echo "got:" 29 | echo "$actual_output" 30 | exit 1 31 | fi 32 | -------------------------------------------------------------------------------- /META.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "holycorn", 3 | "abstract": "PostgreSQL Ruby Foreign Data Wrapper", 4 | "description": "PostgreSQL Ruby Foreign Data Wrapper", 5 | "version": "1.0.0", 6 | "maintainer": [ 7 | "Franck Verrot " 8 | ], 9 | "license": "LGPLv3", 10 | "resources": { 11 | "bugtracker": { 12 | "web": "http://github.com/franckverrot/holycorn/issues/" 13 | }, 14 | "repository": { 15 | "url": "git://github.com/franckverrot/holycorn.git", 16 | "web": "http://github.com/franckverrot/holycorn", 17 | "type": "git" 18 | } 19 | }, 20 | "meta-spec": { 21 | "version": "1.0.0", 22 | "url": "http://pgxn.org/meta/spec.txt" 23 | }, 24 | "tags": [ 25 | "git", 26 | "fdw" 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /tests/Dockerfiles/10: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | MAINTAINER Franck Verrot 3 | 4 | RUN apt-get update -qq && \ 5 | apt-get install -y software-properties-common && \ 6 | add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main 10" 7 | RUN apt-get install -y wget 8 | RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - 9 | RUN apt-get update -qq 10 | RUN apt-get install -y \ 11 | build-essential \ 12 | flex bison \ 13 | git \ 14 | libc6-dev-i386 \ 15 | libpq-dev \ 16 | postgresql-10 \ 17 | postgresql-server-dev-10 \ 18 | rake \ 19 | redis-server 20 | RUN pg_createcluster -p 5433 10 my_cluster 21 | 22 | WORKDIR /holycorn 23 | ADD . /holycorn 24 | 25 | RUN rake build && make install 26 | -------------------------------------------------------------------------------- /tests/Dockerfiles/11: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | MAINTAINER Franck Verrot 3 | 4 | RUN apt-get update -qq && \ 5 | apt-get install -y software-properties-common && \ 6 | add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main 11" 7 | RUN apt-get install -y wget 8 | RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - 9 | RUN apt-get update -qq 10 | RUN apt-get install -y \ 11 | build-essential \ 12 | flex bison \ 13 | git \ 14 | libc6-dev-i386 \ 15 | libpq-dev \ 16 | postgresql-11 \ 17 | postgresql-server-dev-11 \ 18 | rake \ 19 | redis-server 20 | RUN pg_createcluster -p 5433 11 my_cluster 21 | 22 | WORKDIR /holycorn 23 | ADD . /holycorn 24 | 25 | RUN rake build && make install 26 | -------------------------------------------------------------------------------- /tests/Dockerfiles/12: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | MAINTAINER Franck Verrot 3 | 4 | RUN apt-get update -qq && \ 5 | apt-get install -y software-properties-common && \ 6 | add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main 12" 7 | RUN apt-get install -y wget 8 | RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - 9 | RUN apt-get update -qq 10 | RUN apt-get install -y \ 11 | build-essential \ 12 | flex bison \ 13 | git \ 14 | libc6-dev-i386 \ 15 | libpq-dev \ 16 | postgresql-12 \ 17 | postgresql-server-dev-12 \ 18 | rake \ 19 | redis-server 20 | RUN pg_createcluster -p 5433 12 my_cluster 21 | 22 | WORKDIR /holycorn 23 | ADD . /holycorn 24 | 25 | RUN rake build && make install 26 | -------------------------------------------------------------------------------- /tests/Dockerfiles/9.6: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | MAINTAINER Franck Verrot 3 | 4 | RUN apt-get update -qq && \ 5 | apt-get install -y software-properties-common && \ 6 | add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" 7 | RUN apt-get install -y wget 8 | RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - 9 | RUN apt-get update -qq 10 | RUN apt-get install -y \ 11 | build-essential \ 12 | flex bison \ 13 | git \ 14 | libc6-dev-i386 \ 15 | libpq-dev \ 16 | postgresql-9.6 \ 17 | postgresql-server-dev-9.6 \ 18 | rake \ 19 | redis-server 20 | RUN pg_createcluster -p 5433 9.6 my_cluster 21 | 22 | WORKDIR /holycorn 23 | ADD . /holycorn 24 | 25 | RUN rake build && make install 26 | -------------------------------------------------------------------------------- /tests/Dockerfiles/9.4: -------------------------------------------------------------------------------- 1 | FROM ubuntu:xenial 2 | MAINTAINER Franck Verrot 3 | 4 | RUN apt-get update -qq && \ 5 | apt-get install -y software-properties-common && \ 6 | add-apt-repository "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" 7 | RUN apt-get install -y wget 8 | RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - 9 | RUN apt-get update -qq && \ 10 | apt-get install -y \ 11 | build-essential \ 12 | flex bison \ 13 | git \ 14 | libc6-dev-i386 \ 15 | libpq-dev \ 16 | postgresql-9.4 \ 17 | postgresql-server-dev-9.4 \ 18 | rake \ 19 | redis-server 20 | RUN pg_createcluster -p 5433 9.4 my_cluster 21 | 22 | WORKDIR /holycorn 23 | ADD . /holycorn 24 | 25 | RUN rake build && make install 26 | -------------------------------------------------------------------------------- /config/build_config.rb: -------------------------------------------------------------------------------- 1 | def embed_gems(conf) 2 | # Embedded gems 3 | conf.gem :git => 'https://github.com/matsumoto-r/mruby-httprequest.git' 4 | conf.gem :git => 'https://github.com/mattn/mruby-json.git' 5 | conf.gem :git => 'https://github.com/iij/mruby-io.git' 6 | conf.gem :git => 'https://github.com/matsumoto-r/mruby-redis.git' 7 | 8 | # Builtin Foreign Data Wrappers 9 | conf.gem '../../builtin_wrappers/holycorn-redis' 10 | 11 | # Include the default GEMs 12 | conf.gembox 'default' 13 | 14 | [conf.cc, conf.cxx, conf.linker].each do |cc| 15 | cc.flags << "-fPIC" 16 | end 17 | # Include any mruby specific configuration here 18 | # N/A 19 | end 20 | 21 | MRuby::Build.new do |conf| 22 | toolchain :gcc 23 | 24 | enable_debug 25 | 26 | embed_gems(conf) 27 | end 28 | 29 | MRuby::CrossBuild.new('i686-pc-linux-gnu') do |conf| 30 | toolchain :gcc 31 | enable_debug 32 | [conf.cc, conf.cxx, conf.linker].each do |cc| 33 | cc.flags << "-fPIC" 34 | end 35 | embed_gems(conf) 36 | end 37 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | shared: &shared 4 | working_directory: ~/repo 5 | docker: 6 | - image: circleci/ruby:2.6 7 | steps: 8 | - checkout 9 | 10 | - setup_remote_docker: 11 | docker_layer_caching: true 12 | 13 | # run tests! 14 | - run: 15 | name: run tests 16 | command: rake 17 | 18 | jobs: 19 | "PG-9.4": 20 | <<: *shared 21 | environment: 22 | - PG_VERSION=9.4 23 | "PG-9.5": 24 | <<: *shared 25 | environment: 26 | - PG_VERSION=9.5 27 | "PG-9.6": 28 | <<: *shared 29 | environment: 30 | - PG_VERSION=9.6 31 | "PG-10": 32 | <<: *shared 33 | environment: 34 | - PG_VERSION=10 35 | "PG-11": 36 | <<: *shared 37 | environment: 38 | - PG_VERSION=11 39 | "PG-12": 40 | <<: *shared 41 | environment: 42 | - PG_VERSION=12 43 | 44 | workflows: 45 | version: 2 46 | build: 47 | jobs: 48 | - PG-9.4 49 | - PG-9.5 50 | - PG-9.6 51 | - PG-10 52 | - PG-11 53 | # Not yet released, pausing CI for now - PG-12 -------------------------------------------------------------------------------- /builtin_wrappers/holycorn-redis/mrblib/holycorn-redis.rb: -------------------------------------------------------------------------------- 1 | # Known limitations 2 | # 3 | # 1. We should use SCAN instead of KEYS (PR accepted :-D) 4 | # 2. Hashes, Sets and Sorted Sets are not supported yet (ditto) 5 | class HolycornRedis 6 | def initialize(env = {}) 7 | host = env.fetch('host') { raise ArgumentError, 'host not provided' } 8 | port = env.fetch('port') { raise ArgumentError, 'port not provided' } 9 | db = env.fetch('db') { raise ArgumentError, 'db not provided' } 10 | 11 | @r = Redis.new host, port.to_i 12 | @r.select db.to_i 13 | @values = (@r.keys('*') || []).to_enum 14 | end 15 | 16 | def each 17 | if (val = @values.next) 18 | [val.to_s, @r.get(val).to_s] 19 | else 20 | nil 21 | end 22 | end 23 | 24 | def self.import_schema(args) 25 | <<-SCHEMA 26 | CREATE FOREIGN TABLE #{args['local_schema']}.#{args['prefix']}redis_table 27 | ( key text 28 | , value text 29 | ) 30 | SERVER #{args['server_name']} 31 | OPTIONS ( wrapper_class 'HolycornRedis' 32 | , host '#{args['host']}' 33 | , port '#{args['port']}' 34 | , db '#{args['db']}' 35 | ); 36 | SCHEMA 37 | end 38 | self 39 | end 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Holycorn: PostgreSQL multi-purpose Ruby data wrapper [![Travis](https://secure.travis-ci.org/franckverrot/holycorn.png)](http://travis-ci.org/franckverrot/holycorn) 2 | 3 | (m)Ruby + PostgreSQL = <3 4 | 5 | Holycorn makes it easy to implement a Foreign Data Wrapper using Ruby. 6 | 7 | It is based on top of mruby, that provides sandboxing capabilities the regular 8 | Ruby VM "MRI/CRuby" does not implement. 9 | 10 | [![Join the chat at https://gitter.im/franckverrot/holycorn](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/franckverrot/holycorn?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 11 | 12 | 13 | ## Built-in Wrappers 14 | 15 | Holycorn is the combination of the mruby VM, some supporting gems (some basic 16 | libraries and some more advanced ones), and custom code that implement the actual 17 | foreign data wrappers. 18 | 19 | All the following wrappers are currently linked against Holycorn: 20 | 21 | * `Redis`, using the `mruby-redis` gem 22 | 23 | 24 | ## INSTALLATION 25 | 26 | ### Prerequisites 27 | 28 | * PostgreSQL 9.4+ 29 | 30 | ### Setup 31 | 32 | Simply run 33 | 34 | rake 35 | 36 | to vendor and build `mruby`. 37 | 38 | Now that `mruby` is built, building `holycorn` requires to run 39 | 40 | make 41 | 42 | and installing it only requires to run 43 | 44 | make install 45 | 46 | Now connect to PostgreSQL and install the extension: 47 | 48 | λ psql 49 | psql (9.5.14) 50 | Type "help" for help. 51 | 52 | DROP EXTENSION holycorn CASCADE; 53 | CREATE EXTENSION holycorn; 54 | 55 | 56 | ### Using Builtin Foreign Data Wrappers 57 | 58 | #### Manual Creation of Foreign Tables 59 | 60 | A set of builtin FDW are distributed with Holycorn for an easy setup. All one 61 | needs to provide are the options that will allow the FDW to be configured: 62 | 63 | λ psql 64 | psql (9.5.14) 65 | Type "help" for help. 66 | 67 | CREATE SERVER holycorn_server FOREIGN DATA WRAPPER holycorn; 68 | CREATE FOREIGN TABLE redis_table (key text, value text) 69 | SERVER holycorn_server 70 | OPTIONS ( wrapper_class 'HolycornRedis' 71 | , host '127.0.0.1' 72 | , port '6379' 73 | , db '0'); 74 | 75 | 76 | #### Automatic Import using IMPORT FOREIGN SCHEMA 77 | 78 | Alternatively, Holycorn supports `IMPORT FOREIGN SCHEMA` and the same can be 79 | accomplished by using that statement instead: 80 | 81 | λ psql 82 | psql (9.5.14) 83 | 84 | CREATE SERVER holycorn_server FOREIGN DATA WRAPPER holycorn; 85 | IMPORT FOREIGN SCHEMA holycorn_schema 86 | FROM SERVER holycorn_server 87 | INTO holycorn_tables 88 | OPTIONS ( wrapper_class 'HolycornRedis' 89 | , host '127.0.0.1' 90 | , port '6379' 91 | , db '0' 92 | , prefix 'holycorn_' 93 | ); 94 | 95 | Please note that custom data wrappers have to use the `holycorn_schema` schema 96 | as it would otherwise result in an error at runtime. 97 | 98 | Note: `IMPORT FOREIGN SCHEMA` requires us to use a custom target schema and 99 | Holycorn encourages users to use a `prefix` to help avoiding name collisions. 100 | 101 | 102 | #### Access Foreign Tables 103 | 104 | As `Holycorn` doesn't support `INSERT`s yet, let's create some manually: 105 | 106 | ```console 107 | λ redis-cli 108 | 127.0.0.1:6379> select 0 109 | OK 110 | 127.0.0.1:6379> keys * 111 | (empty list or set) 112 | 127.0.0.1:6379> set foo 1 113 | OK 114 | 127.0.0.1:6379> set bar 2 115 | OK 116 | 127.0.0.1:6379> set baz 3 117 | OK 118 | 127.0.0.1:6379> keys * 119 | 1) "bar" 120 | 2) "foo" 121 | 3) "baz" 122 | ``` 123 | 124 | Now that the table has been created and we have some data in Redis, we can 125 | select data from the foreign table. 126 | 127 | ```sql 128 | SELECT * from redis_table; 129 | 130 | key | value 131 | -----+------- 132 | bar | 2 133 | foo | 1 134 | baz | 3 135 | (3 rows) 136 | ``` 137 | 138 | #### Using custom scripts 139 | 140 | Alternatively, custom scripts can be used as the source for a Foreign Data Wrapper: 141 | 142 | ```sql 143 | DROP EXTENSION holycorn CASCADE; 144 | CREATE EXTENSION holycorn; 145 | CREATE SERVER holycorn_server FOREIGN DATA WRAPPER holycorn; 146 | CREATE FOREIGN TABLE holytable (some_date timestamptz) \ 147 | SERVER holycorn_server 148 | OPTIONS (wrapper_path '/tmp/source.rb'); 149 | ``` 150 | 151 | (`IMPORT FOREIGN SCHEMA` is also supported here.) 152 | 153 | And the source file of the wrapper: 154 | 155 | ```ruby 156 | # /tmp/source.rb 157 | class Producer 158 | def initialize(env = {}) # env contains informations provided by Holycorn 159 | end 160 | 161 | def each 162 | @enum ||= Enumerator.new do |y| 163 | 10.times do |t| 164 | y.yield [ Time.now ] 165 | end 166 | end 167 | @enum.next 168 | end 169 | 170 | def import_schema(args = {}) 171 | # Keys are: 172 | # * local_schema: the target schema 173 | # * server_name: name of the foreign data server in-use 174 | # * wrapper_class: name of the current class 175 | # * any other OPTIONS passed to IMPORT FOREIGN SCHEMA 176 | end 177 | 178 | self 179 | end 180 | ``` 181 | 182 | For more details about `#import_schema`, please take a look at the examples 183 | located in in the `builtin_wrappers` directory. 184 | 185 | 186 | Now you can select data out of the wrapper: 187 | 188 | λ psql 189 | psql (9.5.14) 190 | Type "help" for help. 191 | 192 | franck=# SELECT * FROM holytable; 193 | some_date 194 | --------------------- 195 | 2015-06-21 22:39:24 196 | 2015-06-21 22:39:24 197 | 2015-06-21 22:39:24 198 | 2015-06-21 22:39:24 199 | 2015-06-21 22:39:24 200 | 2015-06-21 22:39:24 201 | 2015-06-21 22:39:24 202 | 2015-06-21 22:39:24 203 | 2015-06-21 22:39:24 204 | 2015-06-21 22:39:24 205 | (10 rows) 206 | 207 | Pretty neat. 208 | 209 | # SUPPORTED SCRIPTS 210 | 211 | ## General rules 212 | 213 | Any type of Ruby object can act as a FDW. The only requirements are that it can 214 | receive `.new` (with arity = 1) and return an object that can receive `each` (arity = 0). 215 | 216 | It doesn't **have** to be a `Class`, and there's currently no will to provide a 217 | superclass to be inherited from. 218 | 219 | In future versions, there will be many more callbacks to interact with PG's FDW 220 | infrastructure through `Holycorn`. 221 | 222 | Also, the script can only be a single word - like `MyClass` - as long as 223 | `MyClass` has been defined and exists within your compilation of `mruby`. 224 | 225 | 226 | ## Environment 227 | 228 | A hash is passed by `Holycorn` to the Ruby script. Its current keys are: 229 | 230 | * `PG_VERSION` 231 | * `PG_VERSION_NUM` 232 | * `PACKAGE_STRING` 233 | * `PACKAGE_VERSION` 234 | * `MRUBY_RUBY_VERSION` 235 | * `WRAPPER_PATH` 236 | 237 | 238 | # SUPPORTED TYPES (Ruby => PG) 239 | 240 | ## Builtin types 241 | 242 | * `MRB_TT_FREE` => `null` 243 | * `MRB_TT_FALSE` => `Boolean` 244 | * `MRB_TT_TRUE` => `Boolean` 245 | * `MRB_TT_FIXNUM` => `Int64` 246 | * `MRB_TT_SYMBOL` => `Text` 247 | * `MRB_TT_UNDEF` => Unsupported 248 | * `MRB_TT_FLOAT` => `Float8` 249 | * `MRB_TT_CPTR` => Unsupported 250 | * `MRB_TT_OBJECT` => `Text` (`to_s` is called) 251 | * `MRB_TT_CLASS` => `Text` (`class.to_s` is called) 252 | * `MRB_TT_MODULE` => `Text` (`to_s` is called) 253 | * `MRB_TT_ICLASS` => Unsupported 254 | * `MRB_TT_SCLASS` => Unsupported 255 | * `MRB_TT_PROC` => `Text` (`inspect` is called) 256 | * `MRB_TT_ARRAY` => `Text` (`inspect` is called) 257 | * `MRB_TT_HASH` => `Text` (`inspect` is called) 258 | * `MRB_TT_STRING` => `Text` 259 | * `MRB_TT_RANGE` => `Text` (`inspect` is called) 260 | * `MRB_TT_EXCEPTION` => Unsupported 261 | * `MRB_TT_FILE` => Unsupported 262 | * `MRB_TT_ENV` => Unsupported 263 | * `MRB_TT_DATA` => See "Arbitraty Ruby objects" section 264 | * `MRB_TT_FIBER` => `Text` (`inspect` is called) 265 | * `MRB_TT_MAXDEFINE` => Unsupported 266 | 267 | ## Arbitraty Ruby objects 268 | 269 | * Time (Ruby) => `timestamptz` 270 | 271 | ## CONFIGURATION 272 | 273 | ### Server 274 | 275 | None (yet). 276 | 277 | ### Foreign Table 278 | 279 | Either `wrapper_class` or `wrapper_path` can be used to defined whether an 280 | external script or a built-in wrapper will manage the foreign table. 281 | 282 | * `wrapper_class`: Name of the built-in wrapper class 283 | * `wrapper_path`: Path of a custom script 284 | 285 | In both case, any other option will be pushed down to the wrapper class via the 286 | constructor. 287 | 288 | 289 | ## TODO 290 | 291 | - [ ] Array type 292 | - [ ] JSON type 293 | - [ ] Range type 294 | - [ ] Support PG 9.5's `IMPORT FOREIGN SCHEMA` for easy setup 295 | 296 | ## Note on Patches/Pull Requests 297 | 298 | * Fork the project. 299 | * Make your feature addition or bug fix. 300 | * Add tests for it. This is important so I don't break it in a future version unintentionally. 301 | * Commit, do not mess with version or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull) 302 | * Send me a pull request. Bonus points for topic branches. 303 | 304 | ## LICENSE 305 | 306 | Copyright (c) 2015-2018 Franck Verrot 307 | 308 | Holycorn is an Open Source project licensed under the terms of the LGPLv3 309 | license. Please see for license 310 | text. 311 | 312 | ## AUTHOR 313 | 314 | Franck Verrot, @franckverrot 315 | -------------------------------------------------------------------------------- /holycorn.c: -------------------------------------------------------------------------------- 1 | #include "postgres.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "mruby.h" 9 | #include "mruby/array.h" 10 | #include "mruby/proc.h" 11 | #include "mruby/compile.h" 12 | #include "mruby/string.h" 13 | #include "mruby/range.h" 14 | #include "mruby/hash.h" 15 | 16 | #include "access/htup_details.h" 17 | #include "access/reloptions.h" 18 | #include "access/sysattr.h" 19 | #include "catalog/pg_foreign_table.h" 20 | #include "commands/defrem.h" 21 | #include "commands/explain.h" 22 | #include "commands/vacuum.h" 23 | #include "foreign/fdwapi.h" 24 | #include "foreign/foreign.h" 25 | #include "miscadmin.h" 26 | #include "nodes/makefuncs.h" 27 | #include "optimizer/cost.h" 28 | #include "optimizer/pathnode.h" 29 | #include "optimizer/planmain.h" 30 | #include "optimizer/restrictinfo.h" 31 | #include "optimizer/var.h" 32 | #include "utils/memutils.h" 33 | #include "utils/rel.h" 34 | #include "utils/builtins.h" 35 | #include "utils/timestamp.h" 36 | #include "utils/numeric.h" 37 | #include "plan_state.h" 38 | #include "execution_state.h" 39 | #include "options.h" 40 | 41 | PG_MODULE_MAGIC; 42 | 43 | PG_FUNCTION_INFO_V1(holycorn_handler); 44 | PG_FUNCTION_INFO_V1(holycorn_validator); 45 | 46 | #define POSTGRES_TO_UNIX_EPOCH_DAYS (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) 47 | #define POSTGRES_TO_UNIX_EPOCH_USECS (POSTGRES_TO_UNIX_EPOCH_DAYS * USECS_PER_DAY) 48 | 49 | static void rbGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid); 50 | static void rbGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid); 51 | static ForeignScan *rbGetForeignPlan(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid, 52 | ForeignPath *best_path, List *tlist, List *scan_clauses 53 | #if (PG_VERSION_NUM >= 90500) 54 | , Plan *outer_plan 55 | #endif 56 | ); 57 | static void rbBeginForeignScan(ForeignScanState *node, int eflags); 58 | static void rbExplainForeignScan(ForeignScanState *node, ExplainState *es); 59 | static TupleTableSlot *rbIterateForeignScan(ForeignScanState *node); 60 | static void fileReScanForeignScan(ForeignScanState *node); 61 | static void rbEndForeignScan(ForeignScanState *node); 62 | #if (PG_VERSION_NUM >= 90500) 63 | static List *rbImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid); 64 | #endif 65 | 66 | static void rbGetOptions(Oid foreigntableid, HolycornPlanState *state, List **other_options); 67 | static void estimate_costs(PlannerInfo *root, RelOptInfo *baserel, 68 | HolycornPlanState *fdw_private, 69 | Cost *startup_cost, Cost *total_cost); 70 | 71 | Datum handle_column(mrb_value *, TupleTableSlot *, int, mrb_value); 72 | 73 | Datum holycorn_handler(PG_FUNCTION_ARGS) { 74 | FdwRoutine *fdwroutine = makeNode(FdwRoutine); 75 | 76 | fdwroutine->GetForeignRelSize = rbGetForeignRelSize; 77 | fdwroutine->GetForeignPaths = rbGetForeignPaths; 78 | fdwroutine->GetForeignPlan = rbGetForeignPlan; 79 | 80 | fdwroutine->BeginForeignScan = rbBeginForeignScan; 81 | fdwroutine->IterateForeignScan = rbIterateForeignScan; 82 | fdwroutine->EndForeignScan = rbEndForeignScan; 83 | fdwroutine->ExplainForeignScan = rbExplainForeignScan; 84 | 85 | fdwroutine->ReScanForeignScan = fileReScanForeignScan; 86 | 87 | #if (PG_VERSION_NUM >= 90500) 88 | /* support for IMPORT FOREIGN SCHEMA */ 89 | fdwroutine->ImportForeignSchema = rbImportForeignSchema; 90 | #endif 91 | 92 | PG_RETURN_POINTER(fdwroutine); 93 | } 94 | 95 | Datum holycorn_validator(PG_FUNCTION_ARGS) { 96 | List *options_list = untransformRelOptions(PG_GETARG_DATUM(0)); 97 | Oid catalog = PG_GETARG_OID(1); 98 | List *other_options = NIL; 99 | ListCell *cell; 100 | 101 | /* 102 | * These strings won't be used but they will hold temporary values extracted 103 | * from the foreign table definition 104 | */ 105 | char * wrapper_path = NULL; 106 | char * wrapper_class = NULL; 107 | 108 | foreach(cell, options_list) { 109 | DefElem *def = (DefElem *) lfirst(cell); 110 | 111 | if (strcmp(def->defname, "wrapper_path") == 0) { 112 | if (wrapper_path) 113 | ereport(DEBUG1, 114 | (errcode(ERRCODE_SYNTAX_ERROR), 115 | errmsg("conflicting or redundant options"))); 116 | wrapper_path = defGetString(def); 117 | } 118 | else if (strcmp(def->defname, "wrapper_class") == 0) { 119 | if (wrapper_class) 120 | ereport(DEBUG1, 121 | (errcode(ERRCODE_SYNTAX_ERROR), 122 | errmsg("conflicting or redundant options"))); 123 | wrapper_class = defGetString(def); 124 | } else { 125 | other_options = lappend(other_options, def); 126 | } 127 | } 128 | 129 | if (catalog == ForeignTableRelationId && (wrapper_path != NULL) && (wrapper_class != NULL)) { 130 | elog(ERROR, "[holycorn validator] wrapper_path or wrapper_class are required (and not both) for defining a holycorn foreign table"); 131 | } 132 | 133 | PG_RETURN_VOID(); 134 | } 135 | 136 | static void rbGetOptions(Oid foreigntableid, HolycornPlanState *state, List **other_options) { 137 | ForeignTable *table; 138 | List *options; 139 | ListCell *lc, 140 | *prev; 141 | 142 | table = GetForeignTable(foreigntableid); 143 | 144 | options = NIL; 145 | options = list_concat(options, table->options); 146 | 147 | prev = NULL; 148 | 149 | /* Set default values */ 150 | state->wrapper_path = NULL; 151 | state->wrapper_class = NULL; 152 | 153 | foreach(lc, options) { 154 | DefElem *def = (DefElem *) lfirst(lc); 155 | 156 | if (strcmp(def->defname, "wrapper_path") == 0) { /* Extract the wrapper_path */ 157 | state->wrapper_path = defGetString(def); 158 | options = list_delete_cell(options, lc, prev); 159 | } else if (strcmp(def->defname, "wrapper_class") == 0) { /* Extract the wrapper_class */ 160 | state->wrapper_class = defGetString(def); 161 | options = list_delete_cell(options, lc, prev); 162 | } 163 | 164 | prev = lc; 165 | } 166 | 167 | if (state->wrapper_path != NULL && state->wrapper_class != NULL) { 168 | elog(ERROR, "[holycorn rbGetOptions] wrapper_path or wrapper_class are required (and not both) for defining a holycorn foreign table"); 169 | } 170 | 171 | *other_options = options; 172 | } 173 | 174 | static void rbGetForeignRelSize(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid) { 175 | 176 | HolycornPlanState *fdw_private = (HolycornPlanState *) palloc(sizeof(HolycornPlanState)); 177 | rbGetOptions(foreigntableid, fdw_private, &fdw_private->options); 178 | baserel->fdw_private = (void *) fdw_private; 179 | } 180 | 181 | static void rbGetForeignPaths(PlannerInfo *root, RelOptInfo *baserel, Oid foreigntableid) { 182 | HolycornPlanState *fdw_private = (HolycornPlanState *) baserel->fdw_private; 183 | Cost startup_cost; 184 | Cost total_cost; 185 | List *coptions = NIL; 186 | 187 | /* Estimate costs */ 188 | estimate_costs(root, baserel, fdw_private, &startup_cost, &total_cost); 189 | 190 | /* 191 | * Create a ForeignPath node and add it as only possible path. We use the 192 | * fdw_private list of the path to carry the convert_selectively option; 193 | * it will be propagated into the fdw_private list of the Plan node. 194 | */ 195 | Path * path = (Path*)create_foreignscan_path(root, baserel, 196 | #if PG_VERSION_NUM >= 90600 197 | NULL, /* default pathtarget */ 198 | #endif 199 | baserel->rows, startup_cost, total_cost, 200 | NIL, /* no pathkeys */ 201 | NULL, /* no outer rel either */ 202 | #if PG_VERSION_NUM >= 90500 203 | NULL, /* no extra plan */ 204 | #endif 205 | coptions); 206 | 207 | add_path(baserel, path); 208 | } 209 | 210 | static ForeignScan * rbGetForeignPlan( 211 | PlannerInfo *root, 212 | RelOptInfo *baserel, 213 | Oid foreigntableid, 214 | ForeignPath *best_path, 215 | List *tlist, 216 | List *scan_clauses 217 | #if (PG_VERSION_NUM >= 90500) 218 | , Plan *outer_plan 219 | #endif 220 | ) 221 | { 222 | Index scan_relid = baserel->relid; 223 | scan_clauses = extract_actual_clauses(scan_clauses, false); 224 | 225 | best_path->fdw_private = baserel->fdw_private; 226 | ForeignScan * scan = make_foreignscan( 227 | tlist, 228 | scan_clauses, 229 | scan_relid, 230 | NIL, 231 | best_path->fdw_private 232 | #if PG_VERSION_NUM >= 90500 233 | , NIL 234 | , NIL 235 | , outer_plan 236 | #endif 237 | ); 238 | 239 | return scan; 240 | } 241 | 242 | static void rbExplainForeignScan(ForeignScanState *node, ExplainState *es) { 243 | } 244 | 245 | static void rbBeginForeignScan(ForeignScanState *node, int eflags) { 246 | HolycornExecutionState *exec_state = (HolycornExecutionState *)palloc(sizeof(HolycornExecutionState)); 247 | 248 | ForeignScan *plan = (ForeignScan *)node->ss.ps.plan; 249 | 250 | HolycornPlanState * hps = (HolycornPlanState*)plan->fdw_private; 251 | 252 | exec_state->mrb_state = mrb_open(); 253 | 254 | mrb_value class; 255 | 256 | if(hps->wrapper_path) { 257 | FILE *source = fopen(hps->wrapper_path,"r"); 258 | class = mrb_load_file(exec_state->mrb_state, source); 259 | fclose(source); 260 | } else { 261 | class = mrb_obj_value(mrb_class_get(exec_state->mrb_state, hps->wrapper_class)); 262 | } 263 | 264 | mrb_value params = mrb_hash_new(exec_state->mrb_state); 265 | 266 | #define HASH_SET(hash, key, val) \ 267 | mrb_hash_set(exec_state->mrb_state, hash, mrb_str_new_lit(exec_state->mrb_state, key), val); 268 | 269 | HASH_SET(params, "PG_VERSION", mrb_str_new_lit(exec_state->mrb_state, PG_VERSION)); 270 | HASH_SET(params, "PG_VERSION_NUM", mrb_float_value(exec_state->mrb_state, PG_VERSION_NUM)); 271 | HASH_SET(params, "PACKAGE_STRING", mrb_str_new_lit(exec_state->mrb_state, PACKAGE_STRING)); 272 | HASH_SET(params, "PACKAGE_VERSION", mrb_str_new_lit(exec_state->mrb_state, PACKAGE_VERSION)); 273 | HASH_SET(params, "MRUBY_RUBY_VERSION", mrb_str_new_lit(exec_state->mrb_state, MRUBY_RUBY_VERSION)); 274 | 275 | ListCell *cell; 276 | foreach(cell, hps->options) { 277 | DefElem *def = (DefElem *) lfirst(cell); 278 | 279 | char * key = def->defname; 280 | char * val = defGetString(def); 281 | 282 | mrb_hash_set( \ 283 | exec_state->mrb_state, \ 284 | params, \ 285 | mrb_str_new(exec_state->mrb_state, key, strlen(key)), \ 286 | mrb_str_new(exec_state->mrb_state, val, strlen(val))); 287 | } 288 | 289 | exec_state->iterator = mrb_funcall(exec_state->mrb_state, class, "new", 1, params); 290 | 291 | if (mrb_exception_p(exec_state->iterator)) { 292 | mrb_value message = mrb_funcall(exec_state->mrb_state, exec_state->iterator, "inspect", NULL); 293 | mrb_value pretty_params = mrb_funcall(exec_state->mrb_state, params, "inspect", NULL); 294 | elog(ERROR, 295 | "[holycorn] Instantiating %s raised an exception:\n%s\n (params: %s)\n", 296 | hps->wrapper_class, 297 | RSTRING_PTR(message), 298 | RSTRING_PTR(pretty_params)); 299 | } 300 | 301 | node->fdw_state = (void *) exec_state; 302 | } 303 | 304 | static TupleTableSlot * rbIterateForeignScan(ForeignScanState *node) { 305 | HolycornExecutionState *exec_state = (HolycornExecutionState *) node->fdw_state; 306 | TupleTableSlot *slot = node->ss.ss_ScanTupleSlot; 307 | char * output_str; 308 | 309 | #define StringDatumFromChars(column) \ 310 | output_str = (char *)palloc(sizeof(char) * RSTRING_LEN(column) + 2); /* 2 = 1 prefix (string) + 1 suffix (NULL) */ \ 311 | sprintf(output_str, "s%s\0", RSTRING_PTR(column)); \ 312 | CStringGetDatum(output_str); 313 | 314 | #define INSPECT \ 315 | column = mrb_funcall(exec_state->mrb_state, column, "inspect", 0, NULL); \ 316 | slot->tts_isnull[i] = false; \ 317 | slot->tts_values[i] = StringDatumFromChars(column); 318 | 319 | ExecClearTuple(slot); 320 | 321 | mrb_value output = mrb_funcall(exec_state->mrb_state, exec_state->iterator, "each", 0, NULL); 322 | 323 | if (!mrb_array_p(output)) { 324 | output = mrb_funcall(exec_state->mrb_state, output, "inspect", 0, NULL); 325 | elog(LOG, "#each must provide an array (was %s)", RSTRING_PTR(output)); 326 | return NULL; 327 | } else { 328 | slot->tts_nvalid = RARRAY_LEN(output); 329 | 330 | slot->tts_isempty = false; 331 | slot->tts_isnull = (bool *)palloc(sizeof(bool) * slot->tts_nvalid); 332 | slot->tts_values = (Datum *)palloc(sizeof(Datum) * slot->tts_nvalid); 333 | 334 | if (slot->tts_nvalid <= 0) { //TODO: the size can't be < 0 but being defensive is OK :-) 335 | slot->tts_isempty = true; 336 | } else { 337 | Datum outputDatum; 338 | for(int i = 0; i < slot->tts_nvalid; i++) { 339 | slot->tts_isnull[i] = true; 340 | slot->tts_values[i] = NULL; 341 | mrb_value column = mrb_ary_entry(output, i); 342 | 343 | switch(mrb_type(column)) { 344 | case MRB_TT_FREE: 345 | slot->tts_isnull[i] = true; 346 | break; 347 | case MRB_TT_FALSE: //TODO: use BoolGetDatum 348 | if (!mrb_nil_p(column)) { 349 | slot->tts_isnull[i] = true; 350 | slot->tts_values[i] = BoolGetDatum(false); 351 | } 352 | break; 353 | case MRB_TT_TRUE: 354 | slot->tts_isnull[i] = false; 355 | slot->tts_values[i] = BoolGetDatum(true); 356 | break; 357 | case MRB_TT_FIXNUM: 358 | outputDatum = Int64GetDatum(mrb_fixnum(column)); 359 | slot->tts_isnull[i] = false; 360 | slot->tts_values[i] = outputDatum; 361 | break; 362 | case MRB_TT_SYMBOL: //TODO: use mrb_sym 363 | INSPECT 364 | break; 365 | case MRB_TT_UNDEF: 366 | elog(ERROR,"MRB_TT_UNDEF not supported (yet?)"); 367 | break; 368 | case MRB_TT_FLOAT: 369 | outputDatum = Float8GetDatum(mrb_flo_to_fixnum(column)); 370 | slot->tts_isnull[i] = false; 371 | slot->tts_values[i] = outputDatum; 372 | break; 373 | case MRB_TT_CPTR: 374 | elog(ERROR,"MRB_TT_CPTR not supported (yet?)"); 375 | break; 376 | case MRB_TT_OBJECT: 377 | column = mrb_funcall(exec_state->mrb_state, column, "to_s", 0, NULL); 378 | slot->tts_isnull[i] = false; 379 | slot->tts_values[i] = StringDatumFromChars(column); 380 | break; 381 | case MRB_TT_CLASS: 382 | column = mrb_funcall(exec_state->mrb_state, column, "class", 0, NULL); 383 | column = mrb_funcall(exec_state->mrb_state, column, "to_s", 0, NULL); 384 | slot->tts_isnull[i] = false; 385 | slot->tts_values[i] = StringDatumFromChars(column); 386 | break; 387 | case MRB_TT_MODULE: 388 | column = mrb_funcall(exec_state->mrb_state, column, "to_s", 0, NULL); 389 | slot->tts_isnull[i] = false; 390 | slot->tts_values[i] = StringDatumFromChars(column); 391 | break; 392 | case MRB_TT_ICLASS: 393 | elog(ERROR, "MRB_TT_ICLASS not supported (yet?)"); 394 | break; 395 | case MRB_TT_SCLASS: 396 | elog(ERROR, "MRB_TT_SCLASS not supported (yet?)"); 397 | break; 398 | case MRB_TT_PROC: 399 | INSPECT 400 | break; 401 | case MRB_TT_ARRAY: //TODO: use array type 402 | INSPECT 403 | break; 404 | case MRB_TT_HASH: //TODO: use json type 405 | INSPECT 406 | break; 407 | case MRB_TT_STRING: 408 | slot->tts_isnull[i] = false; 409 | slot->tts_values[i] = StringDatumFromChars(column); 410 | break; 411 | case MRB_TT_RANGE: //TODO: Use the range type 412 | // struct RangeBound *range = (RangeBound*)malloc(sizeof(RangeBound)); 413 | // { 414 | // Datum val; /* the bound value, if any */ 415 | // bool infinite; /* bound is +/- infinity */ 416 | // bool inclusive; /* bound is inclusive (vs exclusive) */ 417 | // bool lower; /* this is the lower (vs upper) bound */ 418 | // } RangeBound; 419 | // mrb_range_ptr(column)->edges->beg, 420 | // mrb_range_ptr(column)->edges->end, 421 | // mrb_range_ptr(column)->excl); 422 | INSPECT 423 | break; 424 | case MRB_TT_EXCEPTION: 425 | elog(ERROR, "MRB_TT_EXCEPTION not supported (yet?)"); 426 | break; 427 | case MRB_TT_FILE: 428 | elog(ERROR, "MRB_TT_FILE not supported (yet?)"); 429 | break; 430 | case MRB_TT_ENV: 431 | elog(ERROR, "MRB_TT_ENV not supported (yet?)"); 432 | break; 433 | case MRB_TT_DATA: 434 | handle_column(exec_state->mrb_state, slot, i, column); 435 | break; 436 | case MRB_TT_FIBER: 437 | INSPECT 438 | break; 439 | case MRB_TT_MAXDEFINE: 440 | elog(ERROR, "MRB_TT_MAX_DEFINE not supported (yet?)"); 441 | break; 442 | default: 443 | slot->tts_isnull[i] = true; 444 | elog(ERROR,"unknown type (0x%x)", mrb_type(column)); 445 | break; 446 | } 447 | ExecStoreVirtualTuple(slot); 448 | } 449 | } 450 | return slot; 451 | } 452 | } 453 | 454 | static void fileReScanForeignScan(ForeignScanState *node) { 455 | } 456 | 457 | static void rbEndForeignScan(ForeignScanState *node) { 458 | HolycornExecutionState *exec_state = (HolycornExecutionState *) node->fdw_state; 459 | mrb_close(exec_state->mrb_state); 460 | } 461 | 462 | static void estimate_costs(PlannerInfo *root, RelOptInfo *baserel, HolycornPlanState *fdw_private, Cost *startup_cost, Cost *total_cost) 463 | { 464 | BlockNumber pages = fdw_private->pages; 465 | double ntuples = fdw_private->ntuples; 466 | Cost run_cost; 467 | Cost cpu_per_tuple; 468 | 469 | *startup_cost = baserel->baserestrictcost.startup; 470 | 471 | // Arbitrary choice to make it very expensive 472 | run_cost = seq_page_cost * pages; 473 | cpu_per_tuple = cpu_tuple_cost * 100 + baserel->baserestrictcost.per_tuple; 474 | run_cost += cpu_per_tuple * ntuples; 475 | *total_cost = *startup_cost + run_cost; 476 | } 477 | 478 | #define IS_A(actual, expected) strcmp(RSTRING_PTR(actual), expected) == 0 479 | 480 | Datum handle_column(mrb_value * mrb, TupleTableSlot *slot, int idx, mrb_value rb_obj) { 481 | char * output_str = NULL; 482 | mrb_value class = mrb_funcall(mrb, rb_obj, "class", 0, NULL); 483 | mrb_value class_name = mrb_funcall(mrb, class, "to_s", 0, NULL); 484 | 485 | if(IS_A(class_name, "Time")) { 486 | rb_obj = mrb_funcall(mrb, rb_obj, "to_i", 0, NULL); 487 | long value = (long)mrb_fixnum(rb_obj); 488 | 489 | long int val = (value * 1000000L) - POSTGRES_TO_UNIX_EPOCH_USECS; 490 | slot->tts_isnull[idx] = false; 491 | slot->tts_values[idx] = Int64GetDatum(val); 492 | } else { 493 | //TODO: serialize the error type in the column? 494 | elog(WARNING, "Unsupported Ruby object (%s). Using #to_s by default", RSTRING_PTR(class_name)); 495 | 496 | slot->tts_isnull[idx] = false; 497 | slot->tts_values[idx] = StringDatumFromChars(class_name); 498 | } 499 | } 500 | #if (PG_VERSION_NUM >= 90500) 501 | static List *rbImportForeignSchema(ImportForeignSchemaStmt *stmt, Oid serverOid) 502 | { 503 | char * wrapper_path = NULL; 504 | char * wrapper_class = NULL; 505 | 506 | mrb_value *state = mrb_open(); 507 | 508 | mrb_value params = mrb_hash_new(state); 509 | 510 | List *commands = NIL; 511 | char **options; 512 | StringInfoData cft_stmt; 513 | if (strcmp(stmt->remote_schema, "holycorn_schema") != 0) { 514 | ereport(ERROR, 515 | (errcode(ERRCODE_FDW_SCHEMA_NOT_FOUND), 516 | errmsg("Foreign schema \"%s\" is invalid. Use `holycorn_schema` instead.", stmt->remote_schema) 517 | )); 518 | } 519 | 520 | ListCell *lc; 521 | foreach(lc, stmt->options) { 522 | DefElem *def = (DefElem *) lfirst(lc); 523 | char * key = def->defname; 524 | char * val = defGetString(def); 525 | 526 | if (strcmp(def->defname, "wrapper_path") == 0) { 527 | if (wrapper_path) 528 | ereport(DEBUG1, 529 | (errcode(ERRCODE_SYNTAX_ERROR), 530 | errmsg("conflicting or redundant options"))); 531 | wrapper_path = defGetString(def); 532 | } 533 | else if (strcmp(def->defname, "wrapper_class") == 0) { 534 | if (wrapper_class) 535 | ereport(DEBUG1, 536 | (errcode(ERRCODE_SYNTAX_ERROR), 537 | errmsg("conflicting or redundant options"))); 538 | wrapper_class = defGetString(def); 539 | } else { 540 | mrb_hash_set( \ 541 | state, \ 542 | params, \ 543 | mrb_str_new(state, key, strlen(key)), \ 544 | mrb_str_new(state, val, strlen(val))); 545 | } 546 | } 547 | 548 | if ((wrapper_path != NULL) && (wrapper_class != NULL)) { 549 | elog(ERROR, "[holycorn import schema] wrapper_path or wrapper_class are required (and not both) for defining a holycorn foreign table"); 550 | } 551 | 552 | mrb_value class; 553 | 554 | if(wrapper_path) { 555 | FILE *source = fopen(wrapper_path,"r"); 556 | class = mrb_load_file(state, source); 557 | fclose(source); 558 | } else { 559 | class = mrb_obj_value(mrb_class_get(state, wrapper_class)); 560 | } 561 | 562 | #define HASH_SET(hash, key, val) \ 563 | mrb_hash_set(state, hash, mrb_str_new_lit(state, key), val); 564 | 565 | HASH_SET(params, "local_schema", mrb_str_new(state, stmt->local_schema, strlen(stmt->local_schema))); 566 | HASH_SET(params, "server_name", mrb_str_new(state, quote_identifier(stmt->server_name), strlen(quote_identifier(stmt->server_name)))); 567 | 568 | mrb_value res = mrb_funcall(state, class, "import_schema", 1, params); 569 | 570 | initStringInfo(&cft_stmt); 571 | appendStringInfo(&cft_stmt, "%s", RSTRING_PTR(res)); 572 | commands = lappend(commands, pstrdup(cft_stmt.data)); 573 | pfree(cft_stmt.data); 574 | return commands; 575 | } 576 | #endif 577 | --------------------------------------------------------------------------------