├── pipelinedb--1.2.0--1.3.0.sql ├── pipelinedb--1.3.0--1.3.1.sql ├── pipelinedb--1.3.1--1.3.2.sql ├── pipelinedb--1.3.2--1.3.3.sql ├── pipelinedb--1.3.3--1.3.4.sql ├── pipelinedb--1.3.5--1.4.0.sql ├── pipelinedb--1.4.0--1.4.1.sql ├── pipelinedb--1.0.0--1.1.0.sql ├── src ├── test │ ├── py │ │ ├── requirements.txt │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── test_worker_limits.py │ │ ├── test_count_distinct.py │ │ ├── test_create_views.py │ │ ├── test_filter.py │ │ ├── test_grouping.py │ │ ├── test_topk_agg.py │ │ ├── test_vacuum.py │ │ ├── test_distinct.py │ │ ├── test_tablespace.py │ │ ├── test_os_aggs.py │ │ ├── test_stream_buffer.py │ │ ├── test_sync.py │ │ ├── test_dist_agg.py │ │ ├── test_freq_agg.py │ │ ├── test_hs_aggs.py │ │ ├── extended.c │ │ ├── test_prepared_inserts.py │ │ ├── test_databases.py │ │ ├── test_typed_streams.py │ │ ├── test_deadlocks.py │ │ ├── test_hll.py │ │ ├── test_pipeline_funcs.py │ │ └── test_cont_transform.py │ └── regress │ │ ├── .gitignore │ │ ├── sql │ │ ├── cont_view_fillfactor.sql │ │ ├── cont_alter.sql │ │ ├── topk.sql │ │ ├── cont_limit.sql │ │ ├── cont_count.sql │ │ ├── matrel_constraints.sql │ │ ├── freq.sql │ │ ├── stream_targets.sql │ │ ├── cont_pk.sql │ │ ├── date_round.sql │ │ ├── dist.sql │ │ ├── cont_index.sql │ │ ├── cont_matrel.sql │ │ ├── part_ttl.sql │ │ ├── cont_activate.sql │ │ ├── cont_part_fillfactor.sql │ │ ├── hash_group.sql │ │ ├── pipeline_stream.sql │ │ ├── bloom.sql │ │ ├── prepared_stream_insert.sql │ │ ├── cont_freq_agg.sql │ │ ├── cont_sw_count.sql │ │ ├── stream_exprs.sql │ │ ├── sw_expiration.sql │ │ ├── cont_dist_agg.sql │ │ ├── stream_insert_subselect.sql │ │ ├── cont_array_agg.sql │ │ ├── hll.sql │ │ ├── cont_bool_agg.sql │ │ ├── cont_view_namespace.sql │ │ ├── cont_grouping_sets.sql │ │ ├── cont_sw_bool_agg.sql │ │ ├── first_values.sql │ │ ├── cont_min_max.sql │ │ ├── cont_complex_types.sql │ │ └── cont_sw_min_max.sql │ │ ├── pipelinedb.conf │ │ ├── expected │ │ ├── cont_alter.out │ │ ├── cont_count.out │ │ ├── cont_view_fillfactor.out │ │ ├── freq.out │ │ ├── matrel_constraints.out │ │ ├── cont_limit.out │ │ ├── topk.out │ │ ├── stream_targets.out │ │ ├── part_ttl.out │ │ ├── cont_activate.out │ │ ├── prepared_stream_insert.out │ │ ├── dist.out │ │ ├── cont_matrel.out │ │ ├── cont_sw_count.out │ │ ├── date_round.out │ │ ├── stream_insert_subselect.out │ │ ├── stream_exprs.out │ │ ├── cont_pk.out │ │ ├── cont_part_fillfactor.out │ │ ├── hll.out │ │ ├── cont_bool_agg.out │ │ └── sw_expiration.out │ │ ├── serial_schedule │ │ └── parallel_schedule └── pipeline_combine.c ├── pipelinedb.control ├── include ├── firstvalues.h ├── topkfuncs.h ├── distfuncs.h ├── bloomfuncs.h ├── hllfuncs.h ├── reaper.h ├── mutator.h ├── tuplestore_scan.h ├── aggfuncs.h ├── json.h ├── commands.h ├── config.h ├── pipeline_combine.h ├── pzmq.h ├── hashfuncs.h ├── physical_group_lookup.h ├── freqfuncs.h ├── transform_receiver.h ├── kv.h ├── copy.h ├── combiner_receiver.h ├── reader.h ├── tdigest.h ├── bloom.h ├── planner.h ├── compat.h ├── cmsketch.h ├── pipeline_stream.h ├── catalog.h ├── matrel.h ├── miscutils.h ├── hll.h ├── stream_fdw.h ├── fss.h └── executor.h ├── NOTICE ├── .gitignore ├── pipelinedb--1.3.4--1.3.5.sql ├── pipelinedb--1.1.0--1.2.0.sql ├── .gitattributes ├── Makefile └── bin └── bootstrap /pipelinedb--1.2.0--1.3.0.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelinedb--1.3.0--1.3.1.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelinedb--1.3.1--1.3.2.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelinedb--1.3.2--1.3.3.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelinedb--1.3.3--1.3.4.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelinedb--1.3.5--1.4.0.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelinedb--1.4.0--1.4.1.sql: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipelinedb--1.0.0--1.1.0.sql: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/test/py/requirements.txt: -------------------------------------------------------------------------------- 1 | pytest==6.2.5 2 | psycopg2-binary==2.9.1 3 | -------------------------------------------------------------------------------- /src/test/regress/.gitignore: -------------------------------------------------------------------------------- 1 | log/ 2 | results/ 3 | expected/pipelinedb.out 4 | sql/pipelinedb.sql 5 | regression.diffs 6 | regression.out 7 | 8 | 9 | -------------------------------------------------------------------------------- /pipelinedb.control: -------------------------------------------------------------------------------- 1 | # PipelineDB extension 2 | comment = 'PipelineDB' 3 | default_version = '1.4.1' 4 | module_pathname = '$libdir/pipelinedb' 5 | relocatable = true 6 | -------------------------------------------------------------------------------- /src/test/py/.gitignore: -------------------------------------------------------------------------------- 1 | /__pycache__ 2 | /.pdb* 3 | /*.pyc 4 | /extended 5 | /analyze_new_cluster.sh 6 | /delete_old_cluster.sh 7 | test_binary_upgrade_data_dir* 8 | test_tablespace/ 9 | .cache 10 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_view_fillfactor.sql: -------------------------------------------------------------------------------- 1 | create foreign table ff_stream (ts timestamp, x int) server pipelinedb; 2 | 3 | create view cv_ff 4 | as select ts, count(*) from ff_stream group by ts; 5 | 6 | \d+ cv_ff_mrel; 7 | -------------------------------------------------------------------------------- /src/pipeline_combine.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * pipeline_combine.c 4 | * 5 | * Copyright (c) 2018, PipelineDB, Inc. 6 | * 7 | * ------------------------------------------------------------------------- 8 | */ 9 | #include "postgres.h" 10 | 11 | Oid PipelineQueryCombineOid = InvalidOid; 12 | -------------------------------------------------------------------------------- /src/test/regress/pipelinedb.conf: -------------------------------------------------------------------------------- 1 | shared_preload_libraries=pipelinedb 2 | max_worker_processes=128 3 | logging_collector = on 4 | log_directory = 'log' 5 | log_statement = 'all' 6 | log_min_messages = info 7 | pipelinedb.stream_insert_level=sync_commit 8 | # pipelinedb.num_workers=4 9 | # pipelinedb.num_combiners=4 10 | log_line_prefix='[%p]' 11 | max_parallel_workers_per_gather=8 12 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_alter.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_alter_stream (x int) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_cont_alter AS SELECT x::int FROM cont_alter_stream; 4 | 5 | ALTER VIEW test_cont_alter RENAME TO lolcat; 6 | ALTER TABLE test_cont_alter_mrel RENAME TO lolcat; 7 | ALTER VIEW test_cont_alter ALTER COLUMN x SET DEFAULT 10; 8 | 9 | DROP FOREIGN TABLE cont_alter_stream CASCADE; 10 | -------------------------------------------------------------------------------- /include/firstvalues.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * firstvalues.h 4 | * 5 | * Interface for first_values aggregate and support functionality 6 | * 7 | * Copyright (c) 2023, Tantor Labs, Inc. 8 | * Copyright (c) 2018, PipelineDB, Inc. 9 | * 10 | *------------------------------------------------------------------------- 11 | */ 12 | #ifndef FIRSTVALUES_H 13 | #define FIRSTVALUES_H 14 | 15 | 16 | #endif 17 | -------------------------------------------------------------------------------- /include/topkfuncs.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * topkfuncs.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef TOPKFUNCS_H 11 | #define TOPKFUNCS_H 12 | 13 | #include "postgres.h" 14 | #include "fmgr.h" 15 | 16 | #define TOPK_TYPENAME "topk" 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /src/test/py/Makefile: -------------------------------------------------------------------------------- 1 | LIBPQ_INCLUDE = $(shell pg_config --includedir) 2 | LIBPQ_LIB = $(shell pg_config --libdir) 3 | 4 | .PHONY: check 5 | 6 | all: extended 7 | 8 | extended: 9 | $(CC) extended.c -I$(LIBPQ_INCLUDE) -L$(LIBPQ_LIB) -lpq -o $@ 10 | 11 | test: all 12 | py.test -v 13 | 14 | clean: 15 | rm -rf ../tmp/.pdb* 16 | rm -rf ./__pycache__ 17 | rm -f extended 18 | rm -rf test_binary_upgrade_data_dir* 19 | rm -rf test_tablespace 20 | rm -rf pg_upgrade* 21 | 22 | -------------------------------------------------------------------------------- /include/distfuncs.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * distfuncs.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef DISTFUNCS_H 11 | #define DISTFUNCS_H 12 | 13 | #include "postgres.h" 14 | #include "fmgr.h" 15 | 16 | #define TDIGEST_TYPENAME "tdigest" 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /src/test/py/test_worker_limits.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | 3 | def test_low_max_worker_processes(pipeline, clean_db): 4 | pipeline.stop() 5 | 6 | try: 7 | pipeline.run({'max_worker_processes': '2'}) 8 | assert False 9 | except Exception as e: 10 | assert 'Background workers failed to start up' in str(e) 11 | 12 | pipeline.stop() 13 | pipeline.run({'max_worker_processes': '8'}) 14 | 15 | pipeline.stop(); 16 | pipeline.run() 17 | -------------------------------------------------------------------------------- /include/bloomfuncs.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * bloomfuncs.h 4 | * Interface for Bloom Filter functions 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef BLOOMFUNCS_H 12 | #define BLOOMFUNCS_H 13 | 14 | #include "postgres.h" 15 | #include "fmgr.h" 16 | 17 | #define BLOOM_TYPENAME "bloom" 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /include/hllfuncs.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * hllfuncs.h 4 | * 5 | * Interface for HyperLogLog-based functions 6 | * 7 | * Copyright (c) 2023, Tantor Labs, Inc. 8 | * Copyright (c) 2018, PipelineDB, Inc. 9 | * 10 | *------------------------------------------------------------------------- 11 | */ 12 | #ifndef HLLFUNCS_H 13 | #define HLLFUNCS_H 14 | 15 | #include "postgres.h" 16 | #include "fmgr.h" 17 | 18 | #define HLL_TYPENAME "hyperloglog" 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /include/reaper.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * reaper.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef REAPER_H 11 | #define REAPER_H 12 | 13 | extern int ttl_expiration_batch_size; 14 | extern int ttl_expiration_threshold; 15 | 16 | int DeleteTTLExpiredRows(RangeVar *cvname, RangeVar *matrel); 17 | 18 | #endif /* REAPER_H */ 19 | -------------------------------------------------------------------------------- /include/mutator.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * mutator.h 4 | * Interface for mutating parse trees 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef PIPELINE_MUTATOR_H 12 | #define PIPELINE_MUTATOR_H 13 | 14 | #include "nodes/parsenodes.h" 15 | 16 | extern Node * raw_expression_tree_mutator(Node *node, Node *(*walker) (), 17 | void *context); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /include/tuplestore_scan.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * tuplestore_scan.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | * ------------------------------------------------------------------------- 9 | */ 10 | #ifndef TUPLESTORE_SCAN_H 11 | #define TUPLESTORE_SCAN_H 12 | 13 | #include "postgres.h" 14 | 15 | #include "optimizer/planner.h" 16 | 17 | extern Node *CreateTuplestoreScanPath(PlannerInfo *root, RelOptInfo *parent, RangeTblEntry *rte); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_alter.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_alter_stream (x int) SERVER pipelinedb; 2 | CREATE VIEW test_cont_alter AS SELECT x::int FROM cont_alter_stream; 3 | ALTER VIEW test_cont_alter RENAME TO lolcat; 4 | ALTER TABLE test_cont_alter_mrel RENAME TO lolcat; 5 | ERROR: cannot rename materialization table "test_cont_alter_mrel" for continuous view "lolcat" 6 | ALTER VIEW test_cont_alter ALTER COLUMN x SET DEFAULT 10; 7 | ERROR: relation "test_cont_alter" does not exist 8 | DROP FOREIGN TABLE cont_alter_stream CASCADE; 9 | NOTICE: drop cascades to view lolcat 10 | -------------------------------------------------------------------------------- /include/aggfuncs.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * aggfuncs.h 4 | * Interface for for aggregate support functions 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef PIPELINEDB_AGGFUNCS_H 12 | #define PIPELINEDB_AGGFUNCS_H 13 | 14 | #include "fmgr.h" 15 | 16 | extern Datum combinable_array_agg_serialize(PG_FUNCTION_ARGS); 17 | extern Datum combinable_array_agg_deserialize(PG_FUNCTION_ARGS); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /src/test/regress/sql/topk.sql: -------------------------------------------------------------------------------- 1 | SELECT topk(topk_agg(x, 10)) FROM generate_series(1, 100) AS x; 2 | SELECT topk_values(topk_agg(x, 10)) FROM ( 3 | SELECT generate_series(1, 100) 4 | UNION ALL 5 | SELECT generate_series(1, 10) 6 | UNION ALL 7 | SELECT generate_series(1, 30)) AS x; 8 | 9 | SELECT topk_freqs(topk_agg(x, 10)) FROM ( 10 | SELECT generate_series(1, 100) 11 | UNION ALL 12 | SELECT generate_series(1, 10) 13 | UNION ALL 14 | SELECT generate_series(1, 30)) AS x; 15 | 16 | SELECT topk(topk_agg(x, 10)) FROM ( 17 | SELECT generate_series(1, 100) 18 | UNION ALL 19 | SELECT generate_series(1, 10) 20 | UNION ALL 21 | SELECT generate_series(1, 30)) AS x; 22 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_limit.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cqlimit_stream (x int) SERVER pipelinedb; 2 | 3 | CREATE VIEW cqlimit AS SELECT x::int FROM cqlimit_stream LIMIT 9 OFFSET 3; 4 | 5 | INSERT INTO cqlimit_stream (x) VALUES (1), (2), (3); 6 | INSERT INTO cqlimit_stream (x) VALUES (4), (5), (6); 7 | INSERT INTO cqlimit_stream (x) VALUES (7), (8), (9); 8 | INSERT INTO cqlimit_stream (x) VALUES (10), (11), (12); 9 | INSERT INTO cqlimit_stream (x) VALUES (13), (14), (15); 10 | INSERT INTO cqlimit_stream (x) VALUES (16), (17), (18); 11 | INSERT INTO cqlimit_stream (x) VALUES (19), (20), (21); 12 | 13 | SELECT * FROM cqlimit ORDER BY x; 14 | SELECT * FROM cqlimit_mrel ORDER BY x; 15 | 16 | DROP FOREIGN TABLE cqlimit_stream CASCADE; 17 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_count.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE stream_cqcount (k text) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_count AS SELECT k::text, COUNT(*) FROM stream_cqcount GROUP BY k; 4 | 5 | INSERT INTO stream_cqcount (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'); 6 | INSERT INTO stream_cqcount (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'), ('y'), ('y'), ('y'), ('y'), ('y'), ('y'); 7 | 8 | SELECT * FROM test_count ORDER BY k; 9 | 10 | INSERT INTO stream_cqcount (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'); 11 | INSERT INTO stream_cqcount (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'), ('y'), ('y'), ('y'), ('y'), ('y'), ('y'); 12 | 13 | SELECT * FROM test_count ORDER BY k; 14 | 15 | DROP FOREIGN TABLE stream_cqcount CASCADE; 16 | -------------------------------------------------------------------------------- /include/json.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * json.h 4 | * Declarations for JSON data type support. 5 | * 6 | * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group 7 | * Portions Copyright (c) 1994, Regents of the University of California 8 | * Portions Copyright (c) 2018, PipelineDB, Inc. 9 | * Portions Copyright (c) 2023, Tantor Labs, Inc. 10 | * 11 | *------------------------------------------------------------------------- 12 | */ 13 | 14 | #ifndef PIPELINE_JSON_H 15 | #define PIPELINE_JSON_H 16 | 17 | #include "lib/stringinfo.h" 18 | 19 | /* functions in json.c */ 20 | extern void escape_json(StringInfo buf, const char *str); 21 | 22 | #endif /* PIPELINE_JSON_H */ 23 | -------------------------------------------------------------------------------- /src/test/py/test_count_distinct.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import random 3 | 4 | 5 | def test_hll_count_distinct(pipeline, clean_db): 6 | """ 7 | Verify that streaming COUNT(DISTINCT) works 8 | """ 9 | pipeline.create_stream('stream0', x='int') 10 | q = 'SELECT COUNT(DISTINCT x::integer) FROM stream0' 11 | pipeline.create_cv('test_count_distinct', q) 12 | 13 | desc = ('x',) 14 | values = [(random.randint(1, 1024),) for n in range(1000)] 15 | 16 | pipeline.insert('stream0', desc, values) 17 | 18 | expected = len(set(values)) 19 | result = pipeline.execute('SELECT count FROM test_count_distinct')[0] 20 | 21 | # Error rate should be well below %2 22 | delta = abs(expected - result['count']) 23 | 24 | assert delta / float(expected) <= 0.02 25 | -------------------------------------------------------------------------------- /include/commands.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * commands.h 4 | * Interface for processing commands 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef COMMANDS_H 12 | #define COMMANDS_H 13 | 14 | #include "tcop/utility.h" 15 | 16 | extern void InstallCommandHooks(void); 17 | 18 | extern void PipelineProcessUtility(PlannedStmt *pstmt, const char *queryString, 19 | bool readOnlyTree, 20 | ProcessUtilityContext context, 21 | ParamListInfo params, 22 | QueryEnvironment *queryEnv, 23 | DestReceiver *dest, QueryCompletion *qc); 24 | 25 | #endif 26 | -------------------------------------------------------------------------------- /include/config.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * config.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | * ------------------------------------------------------------------------- 9 | */ 10 | #ifndef PIPELINEDB_CONFIG 11 | #define PIPELINEDB_CONFIG 12 | 13 | #define PIPELINEDB_EXTENSION_NAME "pipelinedb" 14 | 15 | extern char *pipeline_version_str; 16 | extern char *pipeline_revision_str; 17 | 18 | extern void _PG_init(void); 19 | 20 | extern bool IsCreatePipelineDBCommand(CreateExtensionStmt *ext); 21 | extern bool IsDropPipelineDBCommand(DropStmt *stmt); 22 | extern bool CreatingPipelineDB(void); 23 | extern bool PipelineDBExists(void); 24 | 25 | extern void cont_bgworker_main(Datum arg); 26 | 27 | #endif 28 | 29 | -------------------------------------------------------------------------------- /src/test/regress/sql/matrel_constraints.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE mc_s0 (x integer, y integer) SERVER pipelinedb; 2 | 3 | CREATE VIEW mc_v0 AS SELECT x, sum(y), count(*) FROM mc_s0 GROUP BY x; 4 | 5 | ALTER TABLE mc_v0_mrel ADD CONSTRAINT chk0 CHECK (x > 5); 6 | ALTER TABLE mc_v0_mrel ADD CONSTRAINT chk1 CHECK (count < 3); 7 | ALTER TABLE mc_v0_mrel ADD CONSTRAINT chk2 CHECK (sum < 3); 8 | 9 | INSERT INTO mc_s0 (x, y) SELECT x, 1 AS y FROM generate_series(1, 10) AS x; 10 | 11 | SELECT * FROM mc_v0 ORDER BY x; 12 | 13 | INSERT INTO mc_s0 (x, y) SELECT x, 1 AS y FROM generate_series(1, 10) AS x; 14 | INSERT INTO mc_s0 (x, y) SELECT x, 1 AS y FROM generate_series(1, 10) AS x; 15 | INSERT INTO mc_s0 (x, y) SELECT x, 1 AS y FROM generate_series(1, 10) AS x; 16 | 17 | SELECT * FROM mc_v0 ORDER BY x; 18 | 19 | DROP FOREIGN TABLE mc_s0 CASCADE; -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright 2018, PipelineDB, Inc. 2 | Copyright 2023, Tantor Labs, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | This distribution has a static binary dependency on the ZeroMQ library, 17 | which is available under the GNU Lesser General Public License (version 3) 18 | with a static linking exception (http://zeromq.org/). 19 | -------------------------------------------------------------------------------- /src/test/regress/sql/freq.sql: -------------------------------------------------------------------------------- 1 | SELECT freq(freq_agg(x), 10) FROM generate_series(1, 100) AS x; 2 | SELECT freq(freq_agg(x), 10) FROM (SELECT generate_series(1, 100) AS x UNION ALL SELECT generate_series(1, 100) AS x) _; 3 | SELECT freq(freq_agg(x::text), '10') FROM (SELECT generate_series(1, 100) AS x UNION ALL SELECT generate_series(1, 100) AS x) _; 4 | SELECT freq(freq_agg(x::text), 'not here') FROM (SELECT generate_series(1, 100) AS x UNION ALL SELECT generate_series(1, 100) AS x) _; 5 | SELECT freq(freq_agg(x::text), 'first') FROM (SELECT 'first' AS x FROM generate_series(1, 100) UNION ALL SELECT 'second' AS x FROM generate_series(1, 100)) _; 6 | 7 | SELECT freq(freq_merge_agg, 50) 8 | FROM (SELECT freq_merge_agg(freq_agg) FROM 9 | (SELECT freq_agg(x) FROM generate_series(1, 100) AS x 10 | UNION ALL 11 | SELECT freq_agg(x) FROM generate_series(1, 100) AS x) _) _; -------------------------------------------------------------------------------- /include/pipeline_combine.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * pipeline_combine.h 4 | * Interface for pipelinedb.combine catalog 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | * ------------------------------------------------------------------------- 10 | */ 11 | #ifndef PIPELINE_COMBINE_H 12 | #define PIPELINE_COMBINE_H 13 | 14 | #include "postgres.h" 15 | 16 | typedef struct FormData_pipeline_combine 17 | { 18 | regproc aggfn; 19 | regproc combineaggfn; 20 | } FormData_pipeline_combine; 21 | 22 | typedef FormData_pipeline_combine *Form_pipeline_combine; 23 | 24 | #define Natts_pipeline_combine 2 25 | #define Anum_pipeline_combine_aggfn 1 26 | #define Anum_pipeline_combine_combineaggfn 2 27 | 28 | extern Oid PipelineQueryCombineOid; 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /include/pzmq.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * zmq.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef PZMQ_H 11 | #define PZMQ_H 12 | 13 | #include "postgres.h" 14 | 15 | #define PZMQ_SOCKNAME_STR "ipc://%s/pipeline/zmq/%ld.sock" 16 | 17 | extern char *socket_dir; 18 | 19 | extern void pzmq_init(int max_msg_size, int hwm, int num_destinations, bool enqueue); 20 | extern void pzmq_destroy(void); 21 | 22 | extern void pzmq_bind(uint64 id); 23 | extern void pzmq_connect(uint64 id); 24 | 25 | extern bool pzmq_poll(int timeout); 26 | extern char *pzmq_recv(int *len, int timeout); 27 | extern bool pzmq_send(uint64 id, char *buf, int len, bool wait); 28 | 29 | extern void pzmq_purge_sock_files(void); 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Global excludes across all subdirectories 2 | *.o 3 | *.so 4 | *.so.[0-9] 5 | *.so.[0-9].[0-9] 6 | *.sl 7 | *.sl.[0-9] 8 | *.sl.[0-9].[0-9] 9 | *.dylib 10 | *.dll 11 | *.a 12 | *.mo 13 | *.pot 14 | *.pc 15 | objfiles.txt 16 | .deps/ 17 | *.gcno 18 | *.gcda 19 | *.gcov 20 | *.gcov.out 21 | lcov.info 22 | coverage/ 23 | *.vcproj 24 | *.vcxproj 25 | win32ver.rc 26 | *.exe 27 | lib*dll.def 28 | lib*.pc 29 | *.deb 30 | *.rpm 31 | 32 | # Local excludes in root directory 33 | /GNUmakefile 34 | /config.log 35 | /config.status 36 | /configure.lineno 37 | /pgsql.sln 38 | /pgsql.sln.cache 39 | /Debug/ 40 | /Release/ 41 | /tmp_install 42 | /tmp_check 43 | 44 | src/test/regress/regression.diffs 45 | src/test/regress/regression.out 46 | 47 | # Temp Emacs files 48 | *.*~ 49 | 50 | # Eclipse project files 51 | /.cproject 52 | /.project 53 | /.pydevproject 54 | /.settings/ 55 | 56 | # VSCode project files 57 | /.vscode 58 | /src/test/tmp 59 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_count.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE stream_cqcount (k text) SERVER pipelinedb; 2 | CREATE VIEW test_count AS SELECT k::text, COUNT(*) FROM stream_cqcount GROUP BY k; 3 | INSERT INTO stream_cqcount (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'); 4 | INSERT INTO stream_cqcount (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'), ('y'), ('y'), ('y'), ('y'), ('y'), ('y'); 5 | SELECT * FROM test_count ORDER BY k; 6 | k | count 7 | ---+------- 8 | x | 12 9 | y | 6 10 | (2 rows) 11 | 12 | INSERT INTO stream_cqcount (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'); 13 | INSERT INTO stream_cqcount (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'), ('y'), ('y'), ('y'), ('y'), ('y'), ('y'); 14 | SELECT * FROM test_count ORDER BY k; 15 | k | count 16 | ---+------- 17 | x | 24 18 | y | 12 19 | (2 rows) 20 | 21 | DROP FOREIGN TABLE stream_cqcount CASCADE; 22 | NOTICE: drop cascades to view test_count 23 | -------------------------------------------------------------------------------- /include/hashfuncs.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * hashfuncs.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef HASHFUNCS_H 11 | #define HASHFUNCS_H 12 | 13 | #include "postgres.h" 14 | #include "fmgr.h" 15 | 16 | #define get_combiner_for_shard_hash(hash) ((hash) % num_combiners) 17 | #define is_group_hash_mine(hash) (get_combiner_for_shard_hash(hash) == MyContQueryProc->group_id) 18 | 19 | extern Datum hash_group(PG_FUNCTION_ARGS); 20 | extern Datum ls_hash_group(PG_FUNCTION_ARGS); 21 | extern uint64 slot_hash_group_skip_attr(TupleTableSlot *slot, AttrNumber sw_attno, FuncExpr *hash, FunctionCallInfo fcinfo); 22 | #define slot_hash_group(slot, hash, fcinfo) slot_hash_group_skip_attr((slot), InvalidAttrNumber, (hash), (fcinfo)) 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_view_fillfactor.out: -------------------------------------------------------------------------------- 1 | create foreign table ff_stream (ts timestamp, x int) server pipelinedb; 2 | create view cv_ff 3 | as select ts, count(*) from ff_stream group by ts; 4 | \d+ cv_ff_mrel; 5 | Table "public.cv_ff_mrel" 6 | Column | Type | Collation | Nullable | Default | Storage | Stats target | Description 7 | --------+-----------------------------+-----------+----------+---------+---------+--------------+------------- 8 | ts | timestamp without time zone | | | | plain | | 9 | count | bigint | | | | plain | | 10 | $pk | bigint | | not null | | plain | | 11 | Indexes: 12 | "cv_ff_mrel_pkey" PRIMARY KEY, btree ("$pk") 13 | "cv_ff_mrel_expr_idx" btree (pipelinedb.ls_hash_group(ts)) 14 | Options: fillfactor=50 15 | 16 | -------------------------------------------------------------------------------- /include/physical_group_lookup.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * physical_group_lookup.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef PHYSICAL_GROUP_LOOKUP_H 11 | #define PHYSICAL_GROUP_LOOKUP_H 12 | 13 | #include "postgres.h" 14 | 15 | #include "nodes/execnodes.h" 16 | #include "optimizer/planner.h" 17 | 18 | typedef struct PhysicalTupleData 19 | { 20 | HeapTuple tuple; /* physical tuple belonging to this entry */ 21 | char flags; 22 | } PhysicalTupleData; 23 | 24 | typedef struct PhysicalTupleData *PhysicalTuple; 25 | 26 | extern void SetPhysicalGroupLookupOutput(TupleHashTable output); 27 | extern void SetPhysicalGroupLookupPartitions(List *parts); 28 | extern Node *CreatePhysicalGroupLookupPath(RelOptInfo *joinrel, Path *path); 29 | extern Plan *CreatePhysicalGroupLookupPlan(Plan *outer); 30 | 31 | #endif 32 | -------------------------------------------------------------------------------- /include/freqfuncs.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * freqfuncs.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef FREQHFUNCS_H 11 | #define FREQHFUNCS_H 12 | 13 | #include "postgres.h" 14 | #include "fmgr.h" 15 | 16 | extern Datum cmsketch_print(PG_FUNCTION_ARGS); 17 | extern Datum cmsketch_agg_trans(PG_FUNCTION_ARGS); 18 | extern Datum cmsketch_agg_transp(PG_FUNCTION_ARGS); 19 | extern Datum cmsketch_merge_agg_trans(PG_FUNCTION_ARGS); 20 | extern Datum cmsketch_frequency(PG_FUNCTION_ARGS); 21 | extern Datum cmsketch_total(PG_FUNCTION_ARGS); 22 | extern Datum cmsketch_norm_frequency(PG_FUNCTION_ARGS); 23 | extern Datum cmsketch_empty(PG_FUNCTION_ARGS); 24 | extern Datum cmsketch_emptyp(PG_FUNCTION_ARGS); 25 | extern Datum cmsketch_add(PG_FUNCTION_ARGS); 26 | extern Datum cmsketch_addn(PG_FUNCTION_ARGS); 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/test/regress/sql/stream_targets.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE test_stream_targets_stream (x int) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_stream_targets0 AS SELECT COUNT(*) FROM test_stream_targets_stream; 4 | CREATE VIEW test_stream_targets1 AS SELECT COUNT(*) FROM test_stream_targets_stream; 5 | CREATE VIEW test_stream_targets2 AS SELECT COUNT(*) FROM test_stream_targets_stream; 6 | 7 | INSERT INTO test_stream_targets_stream (x) VALUES (1); 8 | 9 | SET SESSION pipelinedb.stream_targets TO "test_stream_targets1, test_stream_targets2"; 10 | 11 | INSERT INTO test_stream_targets_stream (x) VALUES (1); 12 | 13 | SET SESSION pipelinedb.stream_targets TO test_stream_targets2; 14 | 15 | INSERT INTO test_stream_targets_stream (x) VALUES (1); 16 | 17 | SET SESSION pipelinedb.stream_targets TO DEFAULT; 18 | 19 | INSERT INTO test_stream_targets_stream (x) VALUES (1); 20 | 21 | SELECT * FROM test_stream_targets0; 22 | SELECT * FROM test_stream_targets1; 23 | SELECT * FROM test_stream_targets2; 24 | 25 | DROP FOREIGN TABLE test_stream_targets_stream CASCADE; 26 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_pk.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE test_pk_stream (x int) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_pk0 WITH (pk='x') AS SELECT x::integer, COUNT(*) FROM test_pk_stream GROUP BY x; 4 | \d+ test_pk0_mrel 5 | 6 | INSERT INTO test_pk_stream (x) SELECT generate_series(1, 20) AS x; 7 | INSERT INTO test_pk_stream (x) SELECT generate_series(1, 30) AS x; 8 | 9 | SELECT * FROM test_pk0 ORDER BY x; 10 | DROP VIEW test_pk0; 11 | 12 | CREATE VIEW test_pk1 WITH (pk='count') AS SELECT x::integer, COUNT(*) FROM test_pk_stream GROUP BY x; 13 | \d+ test_pk1_mrel 14 | 15 | INSERT INTO test_pk_stream (x) VALUES (0); 16 | INSERT INTO test_pk_stream (x) VALUES (0); 17 | INSERT INTO test_pk_stream (x) VALUES (1); 18 | INSERT INTO test_pk_stream (x) VALUES (1); 19 | 20 | SELECT * FROM test_pk1 ORDER BY x; 21 | 22 | DROP VIEW test_pk1; 23 | 24 | CREATE VIEW wrong_arg_type WITH (pk=1) AS SELECT COUNT(*) FROM test_pk_stream; 25 | CREATE VIEW no_column WITH (pk='not_here') AS SELECT COUNT(*) FROM test_pk_stream; 26 | 27 | DROP FOREIGN TABLE test_pk_stream CASCADE; 28 | -------------------------------------------------------------------------------- /src/test/py/test_create_views.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | from base import pipeline, clean_db 4 | 5 | # XXX(usmanm): update this, if it ever changes! 6 | MAX_CQS = 1024 7 | 8 | def test_create_views(pipeline, clean_db): 9 | cvs = [] 10 | pipeline.create_stream('stream0', x='int') 11 | q = 'SELECT count(*) FROM stream0' 12 | 13 | for i in range(1, MAX_CQS): 14 | cvs.append('cv_%d' % i) 15 | pipeline.create_cv(cvs[-1], q) 16 | 17 | try: 18 | pipeline.create_cv('cv_fail', q) 19 | assert False 20 | except Exception as e: 21 | assert 'maximum number of continuous queries exceeded' in e.pgerror 22 | 23 | ids = [r['id'] for r in 24 | pipeline.execute('SELECT id FROM pipelinedb.get_views()')] 25 | 26 | assert len(set(ids)) == len(ids) 27 | assert set(ids) == set(range(1, MAX_CQS)) 28 | 29 | num_remove = random.randint(128, 512) 30 | 31 | for _ in range(num_remove): 32 | pipeline.drop_cv(cvs.pop()) 33 | 34 | for _ in range(num_remove): 35 | cvs.append('cv_%d' % (len(cvs) + 1)) 36 | pipeline.create_cv(cvs[-1], q) 37 | -------------------------------------------------------------------------------- /src/test/regress/sql/date_round.sql: -------------------------------------------------------------------------------- 1 | -- nulls 2 | SELECT date_round(NULL, NULL); 3 | SELECT date_round(NULL, INTERVAL '5 minute'); 4 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', NULL); 5 | 6 | -- valid floors 7 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '15 second'); 8 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '10 minute'); 9 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '3 hour'); 10 | SELECT date_round(TIMESTAMP '2001-02-18 20:38:40', INTERVAL '5 day'); 11 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '2 month'); 12 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '7 year'); 13 | 14 | -- invalid floors 15 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '150 second'); 16 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '65 minute'); 17 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '48 hour'); 18 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '36 day'); 19 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '18 month'); 20 | -------------------------------------------------------------------------------- /src/test/regress/sql/dist.sql: -------------------------------------------------------------------------------- 1 | SELECT dist_quantile(dist_agg(x), 0.50) FROM generate_series(1, 1000) AS x; 2 | SELECT dist_quantile(dist_agg(x), 0.10) FROM generate_series(1, 1000) AS x; 3 | SELECT dist_quantile(dist_agg(x), 0.05) FROM generate_series(1, 1000) AS x; 4 | SELECT dist_quantile(dist_agg(x), 0.90) FROM generate_series(1, 1000) AS x; 5 | SELECT dist_quantile(dist_agg(x), 0.99) FROM generate_series(1, 1000) AS x; 6 | 7 | SELECT dist_cdf(dist_agg(x), 1) FROM generate_series(1, 1000) AS x; 8 | SELECT dist_cdf(dist_agg(x), 5) FROM generate_series(1, 1000) AS x; 9 | SELECT dist_cdf(dist_agg(x), 10) FROM generate_series(1, 1000) AS x; 10 | SELECT dist_cdf(dist_agg(x), 500) FROM generate_series(1, 1000) AS x; 11 | SELECT dist_cdf(dist_agg(x), 999) FROM generate_series(1, 1000) AS x; 12 | SELECT dist_cdf(dist_agg(x), 1000) FROM generate_series(1, 1000) AS x; 13 | SELECT dist_cdf(dist_agg(x), 2000) FROM generate_series(1, 1000) AS x; 14 | 15 | SELECT dist_quantile(dist_add(dist_agg(x), 10000), 0.99) FROM generate_series(1, 1000) AS x; 16 | SELECT dist_cdf(dist_add(dist_agg(x), 10000), 2000) FROM generate_series(1, 1000) AS x; 17 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_index.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_idx_stream (x int, y int) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_cont_index0 AS SELECT x::integer, COUNT(*), AVG(x) FROM cont_idx_stream GROUP BY x; 4 | 5 | CREATE INDEX test_ci_idx0 ON test_cont_index0 (x); 6 | \d+ test_cont_index0_mrel 7 | 8 | CREATE INDEX test_ci_idx1 ON test_cont_index0 (avg); 9 | \d+ test_cont_index0_mrel 10 | 11 | CREATE INDEX test_ci_idx2 ON test_cont_index0 (x, avg); 12 | \d+ test_cont_index0_mrel 13 | 14 | DROP VIEW test_cont_index0; 15 | 16 | CREATE VIEW test_cont_index1 WITH (sw = '1 hour') AS SELECT x::integer, y::integer, COUNT(*), AVG(x) FROM cont_idx_stream GROUP BY x, y; 17 | 18 | CREATE INDEX test_ci_idx0 ON test_cont_index1 (x); 19 | \d+ test_cont_index1_mrel 20 | 21 | CREATE INDEX test_ci_idx1 ON test_cont_index1 (avg); 22 | \d+ test_cont_index1_mrel 23 | 24 | CREATE INDEX test_ci_idx2 ON test_cont_index1 (x, avg); 25 | \d+ test_cont_index1_mrel 26 | 27 | CREATE INDEX test_ci_idx3 ON test_cont_index1 (x, y); 28 | \d+ test_cont_index1_mrel 29 | 30 | DROP VIEW test_cont_index1; 31 | 32 | DROP FOREIGN TABLE cont_idx_stream CASCADE; 33 | -------------------------------------------------------------------------------- /include/transform_receiver.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * transform_receiver.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef TRANSFORM_RECEIVER_H 11 | #define TRANSFORM_RECEIVER_H 12 | 13 | #include "tcop/dest.h" 14 | #include "executor.h" 15 | 16 | typedef struct TransformReceiver 17 | { 18 | BatchReceiver base; 19 | ContQuery *cont_query; 20 | ContExecutor *cont_exec; 21 | Relation tg_rel; 22 | bool os_has_readers; 23 | FunctionCallInfo trig_fcinfo; 24 | 25 | /* only used by the optimized code path for pipeline_stream_insert */ 26 | HeapTuple *tups; 27 | int nmaxtups; 28 | int ntups; 29 | 30 | AttrNumber *osrel_attrs; 31 | bool needs_alignment; 32 | } TransformReceiver; 33 | 34 | typedef void (*TransformFlushFunc) (void); 35 | extern TransformFlushFunc TransformFlushHook; 36 | 37 | extern BatchReceiver *CreateTransformReceiver(ContExecutor *exec, ContQuery *query, Tuplestorestate *buffer); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_matrel.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_matrel_stream (x int) SERVER pipelinedb; 2 | CREATE VIEW cont_matrel AS SELECT COUNT(*) FROM cont_matrel_stream; 3 | 4 | INSERT INTO cont_matrel_stream (x) VALUES (1); 5 | SELECT * FROM cont_matrel; 6 | 7 | INSERT INTO cont_matrel_mrel (count) VALUES (1); 8 | UPDATE cont_matrel_mrel SET count = 2; 9 | DELETE FROM cont_matrel_mrel; 10 | 11 | SET pipelinedb.matrels_writable TO ON; 12 | 13 | UPDATE cont_matrel_mrel SET count = 2; 14 | SELECT * FROM cont_matrel; 15 | INSERT INTO cont_matrel_stream (x) VALUES (1); 16 | SELECT * FROM cont_matrel; 17 | 18 | DELETE FROM cont_matrel_mrel; 19 | SELECT * FROM cont_matrel; 20 | INSERT INTO cont_matrel_stream (x) VALUES (1); 21 | SELECT * FROM cont_matrel; 22 | 23 | SELECT pipelinedb.truncate_continuous_view('cont_matrel'); 24 | INSERT INTO cont_matrel_mrel (count, "$pk") VALUES (5, 1); 25 | INSERT INTO cont_matrel_mrel (count, "$pk") VALUES (10, 1); 26 | SELECT * FROM cont_matrel; 27 | INSERT INTO cont_matrel_stream (x) VALUES (1); 28 | SELECT * FROM cont_matrel; 29 | 30 | DROP VIEW cont_matrel; 31 | DROP FOREIGN TABLE cont_matrel_stream CASCADE; 32 | -------------------------------------------------------------------------------- /pipelinedb--1.3.4--1.3.5.sql: -------------------------------------------------------------------------------- 1 | DROP AGGREGATE IF EXISTS combine_interval_avg(internal); 2 | DROP AGGREGATE IF EXISTS partial_combine_interval_avg(internal); 3 | DROP AGGREGATE IF EXISTS combine_interval_sum(internal); 4 | 5 | CREATE AGGREGATE combine_interval_avg(internal) ( 6 | sfunc = interval_avg_combine, 7 | stype = internal, 8 | finalfunc = interval_avg, 9 | combinefunc = interval_avg_combine, 10 | serialfunc = interval_avg_serialize, 11 | deserialfunc = interval_avg_deserialize, 12 | parallel = safe 13 | ); 14 | 15 | CREATE AGGREGATE partial_combine_interval_avg(internal) ( 16 | sfunc = interval_avg_combine, 17 | stype = internal, 18 | finalfunc = interval_avg, 19 | combinefunc = interval_avg_combine, 20 | serialfunc = interval_avg_serialize, 21 | deserialfunc = interval_avg_deserialize, 22 | parallel = safe 23 | ); 24 | 25 | CREATE AGGREGATE combine_interval_sum(internal) ( 26 | sfunc = interval_avg_combine, 27 | stype = internal, 28 | finalfunc = interval_sum, 29 | combinefunc = interval_avg_combine, 30 | serialfunc = interval_avg_serialize, 31 | deserialfunc = interval_avg_deserialize, 32 | parallel = safe 33 | ); 34 | -------------------------------------------------------------------------------- /src/test/regress/expected/freq.out: -------------------------------------------------------------------------------- 1 | SELECT freq(freq_agg(x), 10) FROM generate_series(1, 100) AS x; 2 | freq 3 | ------ 4 | 1 5 | (1 row) 6 | 7 | SELECT freq(freq_agg(x), 10) FROM (SELECT generate_series(1, 100) AS x UNION ALL SELECT generate_series(1, 100) AS x) _; 8 | freq 9 | ------ 10 | 2 11 | (1 row) 12 | 13 | SELECT freq(freq_agg(x::text), '10') FROM (SELECT generate_series(1, 100) AS x UNION ALL SELECT generate_series(1, 100) AS x) _; 14 | freq 15 | ------ 16 | 2 17 | (1 row) 18 | 19 | SELECT freq(freq_agg(x::text), 'not here') FROM (SELECT generate_series(1, 100) AS x UNION ALL SELECT generate_series(1, 100) AS x) _; 20 | freq 21 | ------ 22 | 0 23 | (1 row) 24 | 25 | SELECT freq(freq_agg(x::text), 'first') FROM (SELECT 'first' AS x FROM generate_series(1, 100) UNION ALL SELECT 'second' AS x FROM generate_series(1, 100)) _; 26 | freq 27 | ------ 28 | 100 29 | (1 row) 30 | 31 | SELECT freq(freq_merge_agg, 50) 32 | FROM (SELECT freq_merge_agg(freq_agg) FROM 33 | (SELECT freq_agg(x) FROM generate_series(1, 100) AS x 34 | UNION ALL 35 | SELECT freq_agg(x) FROM generate_series(1, 100) AS x) _) _; 36 | freq 37 | ------ 38 | 2 39 | (1 row) 40 | 41 | -------------------------------------------------------------------------------- /src/test/regress/expected/matrel_constraints.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE mc_s0 (x integer, y integer) SERVER pipelinedb; 2 | CREATE VIEW mc_v0 AS SELECT x, sum(y), count(*) FROM mc_s0 GROUP BY x; 3 | ALTER TABLE mc_v0_mrel ADD CONSTRAINT chk0 CHECK (x > 5); 4 | ALTER TABLE mc_v0_mrel ADD CONSTRAINT chk1 CHECK (count < 3); 5 | ALTER TABLE mc_v0_mrel ADD CONSTRAINT chk2 CHECK (sum < 3); 6 | INSERT INTO mc_s0 (x, y) SELECT x, 1 AS y FROM generate_series(1, 10) AS x; 7 | SELECT * FROM mc_v0 ORDER BY x; 8 | x | sum | count 9 | ----+-----+------- 10 | 6 | 1 | 1 11 | 7 | 1 | 1 12 | 8 | 1 | 1 13 | 9 | 1 | 1 14 | 10 | 1 | 1 15 | (5 rows) 16 | 17 | INSERT INTO mc_s0 (x, y) SELECT x, 1 AS y FROM generate_series(1, 10) AS x; 18 | INSERT INTO mc_s0 (x, y) SELECT x, 1 AS y FROM generate_series(1, 10) AS x; 19 | INSERT INTO mc_s0 (x, y) SELECT x, 1 AS y FROM generate_series(1, 10) AS x; 20 | SELECT * FROM mc_v0 ORDER BY x; 21 | x | sum | count 22 | ----+-----+------- 23 | 6 | 2 | 2 24 | 7 | 2 | 2 25 | 8 | 2 | 2 26 | 9 | 2 | 2 27 | 10 | 2 | 2 28 | (5 rows) 29 | 30 | DROP FOREIGN TABLE mc_s0 CASCADE; 31 | NOTICE: drop cascades to view mc_v0 32 | -------------------------------------------------------------------------------- /src/test/py/test_filter.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | 3 | 4 | def test_filter_clause(pipeline, clean_db): 5 | """ 6 | Verify that FILTER clauses work on aggregates and sliding window aggregates 7 | """ 8 | pipeline.create_stream('test_filter_stream', x='int') 9 | q = """ 10 | SELECT SUM(x::int) FILTER (WHERE mod(x, 2) = 0) AS sum2, SUM(x::int) FILTER (WHERE mod(x, 3) = 0) AS sum3 FROM test_filter_stream 11 | """ 12 | sw = """ 13 | WHERE arrival_timestamp > clock_timestamp() - interval '30 second' 14 | """ 15 | pipeline.create_cv('test_filter', q) 16 | pipeline.create_cv('test_filter_sw', '%s %s' % (q, sw)) 17 | 18 | desc = ('x', ) 19 | rows = [] 20 | for n in range(1000): 21 | rows.append((n, )) 22 | 23 | pipeline.insert('test_filter_stream', desc, rows) 24 | 25 | sum2 = sum(filter(lambda x: x % 2 == 0, map(lambda x: x[0], rows))) 26 | sum3 = sum(filter(lambda x: x % 3 == 0, map(lambda x: x[0], rows))) 27 | 28 | result1 = pipeline.execute('SELECT * FROM test_filter')[0] 29 | result2 = pipeline.execute('SELECT * FROM test_filter_sw')[0] 30 | 31 | assert result1['sum2'] == result2['sum2'] == sum2 32 | assert result1['sum3'] == result2['sum3'] == sum3 33 | -------------------------------------------------------------------------------- /src/test/py/test_grouping.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import random 3 | 4 | 5 | def test_null_groups(pipeline, clean_db): 6 | """ 7 | Verify that null group columns are considered equal 8 | """ 9 | pipeline.create_stream('s', x='int', y='int', z='int') 10 | q = """ 11 | SELECT x::integer, y::integer, z::integer, COUNT(*) FROM s 12 | GROUP BY x, y, z; 13 | """ 14 | desc = ('x', 'y', 'z') 15 | pipeline.create_cv('test_null_groups', q) 16 | pipeline.create_table('test_null_groups_t', x='integer', y='integer', z='integer') 17 | 18 | rows = [] 19 | for n in range(10000): 20 | vals = list(random.randint(0, 10) for n in range(3)) 21 | vals = map(lambda n: random.random() > 0.1 and n or None, vals) 22 | rows.append(tuple(vals)) 23 | 24 | pipeline.insert('s', desc, rows) 25 | pipeline.insert('test_null_groups_t', desc, rows) 26 | 27 | table_q = """ 28 | SELECT x, y, z, COUNT(*) FROM test_null_groups_t 29 | GROUP BY x, y, z ORDER BY x, y, z; 30 | """ 31 | expected = pipeline.execute(table_q) 32 | result = pipeline.execute('SELECT x, y, z, count FROM test_null_groups ORDER BY x, y, z') 33 | 34 | for r, e in zip(result, expected): 35 | assert r == e 36 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_limit.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cqlimit_stream (x int) SERVER pipelinedb; 2 | CREATE VIEW cqlimit AS SELECT x::int FROM cqlimit_stream LIMIT 9 OFFSET 3; 3 | INSERT INTO cqlimit_stream (x) VALUES (1), (2), (3); 4 | INSERT INTO cqlimit_stream (x) VALUES (4), (5), (6); 5 | INSERT INTO cqlimit_stream (x) VALUES (7), (8), (9); 6 | INSERT INTO cqlimit_stream (x) VALUES (10), (11), (12); 7 | INSERT INTO cqlimit_stream (x) VALUES (13), (14), (15); 8 | INSERT INTO cqlimit_stream (x) VALUES (16), (17), (18); 9 | INSERT INTO cqlimit_stream (x) VALUES (19), (20), (21); 10 | SELECT * FROM cqlimit ORDER BY x; 11 | x 12 | ---- 13 | 4 14 | 5 15 | 6 16 | 7 17 | 8 18 | 9 19 | 10 20 | 11 21 | 12 22 | (9 rows) 23 | 24 | SELECT * FROM cqlimit_mrel ORDER BY x; 25 | x | $pk 26 | ----+----- 27 | 1 | 1 28 | 2 | 2 29 | 3 | 3 30 | 4 | 4 31 | 5 | 5 32 | 6 | 6 33 | 7 | 7 34 | 8 | 8 35 | 9 | 9 36 | 10 | 10 37 | 11 | 11 38 | 12 | 12 39 | 13 | 13 40 | 14 | 14 41 | 15 | 15 42 | 16 | 16 43 | 17 | 17 44 | 18 | 18 45 | 19 | 19 46 | 20 | 20 47 | 21 | 21 48 | (21 rows) 49 | 50 | DROP FOREIGN TABLE cqlimit_stream CASCADE; 51 | NOTICE: drop cascades to view cqlimit 52 | -------------------------------------------------------------------------------- /include/kv.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * kv.h 4 | * Simple key-value pair type implementation. 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef KV_H 12 | #define KV_H 13 | 14 | #include "c.h" 15 | 16 | #include "fmgr.h" 17 | 18 | typedef struct KeyValue 19 | { 20 | uint32 vl_len_; 21 | char flags; 22 | int klen; 23 | int vlen; 24 | Oid key_collation; 25 | Oid key_type; 26 | Oid value_type; 27 | Datum key; 28 | Datum value; 29 | } KeyValue; 30 | 31 | #define KV_KEY_NULL 0x1 32 | #define KV_VALUE_NULL 0x2 33 | 34 | #define KV_SET_KEY_NULL(kv) ((kv)->flags |= KV_KEY_NULL) 35 | #define KV_KEY_IS_NULL(kv) ((kv)->flags & KV_KEY_NULL) 36 | 37 | #define KV_SET_VALUE_NULL(kv) ((kv)->flags |= KV_VALUE_NULL) 38 | #define KV_VALUE_IS_NULL(kv) ((kv)->flags & KV_VALUE_NULL) 39 | 40 | Datum keyed_min_trans(PG_FUNCTION_ARGS); 41 | Datum keyed_max_trans(PG_FUNCTION_ARGS); 42 | Datum keyed_min_max_finalize(PG_FUNCTION_ARGS); 43 | Datum keyed_min_combine(PG_FUNCTION_ARGS); 44 | Datum keyed_max_combine(PG_FUNCTION_ARGS); 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /include/copy.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * copy.h 4 | * COPY for streams 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Portions Copyright (c) 2018, PipelineDB, Inc. 8 | * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group 9 | * Portions Copyright (c) 1994, Regents of the University of California 10 | * 11 | *------------------------------------------------------------------------- 12 | */ 13 | #ifndef PIPELINE_COPY_H 14 | #define PIPELINE_COPY_H 15 | 16 | #include "commands/copy.h" 17 | #include "nodes/execnodes.h" 18 | #include "nodes/parsenodes.h" 19 | #include "parser/parse_node.h" 20 | #include "tcop/dest.h" 21 | 22 | extern copy_data_source_cb stream_copy_hook; 23 | 24 | extern void DoStreamCopy(ParseState *state, const CopyStmt *stmt, 25 | int stmt_location, int stmt_len, 26 | uint64 *processed); 27 | 28 | extern uint64 CopyStreamFrom(CopyFromState cstate); 29 | extern bool NextCopyStreamFrom(CopyFromState cstate, ExprContext *econtext, 30 | Datum *values, bool *nulls, Oid *tupleOid); 31 | extern void ProcessStreamCopyOptions(ParseState *pstate, CopyFromState cstate, bool is_from, List *options); 32 | extern Expr *expression_planner(Expr *expr); 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /src/test/regress/sql/part_ttl.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE ttl_stream (x integer) SERVER pipelinedb; 2 | 3 | CREATE VIEW ttl0 WITH (ttl='3 seconds', ttl_column='ts', 4 | partition_by='ts', partition_duration='1 second') 5 | AS SELECT arrival_timestamp AS ts, x FROM ttl_stream; 6 | 7 | SELECT c.relname, ttl, ttl_attno FROM pipelinedb.cont_query cq 8 | JOIN pg_class c ON c.oid = cq.relid WHERE c.relname = 'ttl0' 9 | ORDER BY c.relname; 10 | 11 | INSERT INTO ttl_stream (x) VALUES (0); 12 | INSERT INTO ttl_stream (x) VALUES (1); 13 | INSERT INTO ttl_stream (x) VALUES (2); 14 | 15 | SELECT x, "$pk" FROM ttl0_mrel ORDER BY ts; 16 | 17 | SELECT pg_sleep(3); 18 | 19 | SELECT 0 * pipelinedb.ttl_expire('ttl0'); 20 | 21 | SELECT x, "$pk" FROM ttl0_mrel ORDER BY ts; 22 | 23 | INSERT INTO ttl_stream (x) VALUES (0); 24 | INSERT INTO ttl_stream (x) VALUES (1); 25 | INSERT INTO ttl_stream (x) VALUES (2); 26 | 27 | SELECT x, "$pk" FROM ttl0_mrel ORDER BY ts; 28 | 29 | SELECT pg_sleep(3); 30 | SELECT 0 * pipelinedb.ttl_expire('ttl0'); 31 | 32 | SELECT x, "$pk" FROM ttl0_mrel ORDER BY ts; 33 | 34 | DROP VIEW ttl0; 35 | 36 | CREATE VIEW ttl2 WITH (partition_by='ts', partition_duration='1 second') 37 | AS SELECT arrival_timestamp AS ts, x FROM ttl_stream; 38 | 39 | DROP VIEW ttl2; 40 | 41 | DROP FOREIGN TABLE ttl_stream CASCADE; 42 | -------------------------------------------------------------------------------- /src/test/regress/expected/topk.out: -------------------------------------------------------------------------------- 1 | SELECT topk(topk_agg(x, 10)) FROM generate_series(1, 100) AS x; 2 | topk 3 | -------- 4 | (40,2) 5 | (45,2) 6 | (47,2) 7 | (56,2) 8 | (58,2) 9 | (79,2) 10 | (80,2) 11 | (82,2) 12 | (83,2) 13 | (86,2) 14 | (10 rows) 15 | 16 | SELECT topk_values(topk_agg(x, 10)) FROM ( 17 | SELECT generate_series(1, 100) 18 | UNION ALL 19 | SELECT generate_series(1, 10) 20 | UNION ALL 21 | SELECT generate_series(1, 30)) AS x; 22 | topk_values 23 | -------------------------------------------- 24 | {(1),(4),(8),(5),(6),(3),(2),(7),(9),(10)} 25 | (1 row) 26 | 27 | SELECT topk_freqs(topk_agg(x, 10)) FROM ( 28 | SELECT generate_series(1, 100) 29 | UNION ALL 30 | SELECT generate_series(1, 10) 31 | UNION ALL 32 | SELECT generate_series(1, 30)) AS x; 33 | topk_freqs 34 | ----------------------- 35 | {3,3,3,3,3,3,3,3,3,3} 36 | (1 row) 37 | 38 | SELECT topk(topk_agg(x, 10)) FROM ( 39 | SELECT generate_series(1, 100) 40 | UNION ALL 41 | SELECT generate_series(1, 10) 42 | UNION ALL 43 | SELECT generate_series(1, 30)) AS x; 44 | topk 45 | ------------ 46 | ("(1)",3) 47 | ("(4)",3) 48 | ("(8)",3) 49 | ("(5)",3) 50 | ("(6)",3) 51 | ("(3)",3) 52 | ("(2)",3) 53 | ("(7)",3) 54 | ("(9)",3) 55 | ("(10)",3) 56 | (10 rows) 57 | 58 | -------------------------------------------------------------------------------- /src/test/py/test_topk_agg.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | 3 | import random 4 | 5 | def get_geometric_dist(items): 6 | values = map(lambda i: [i] * pow(2, items.index(i)), items) 7 | values = sum(values, []) 8 | random.shuffle(values) 9 | return values 10 | 11 | 12 | def test_topk_agg(pipeline, clean_db): 13 | pipeline.create_stream('test_fss_stream', x='int', k='text') 14 | q = """ 15 | SELECT k::text, topk_agg(x::int, 5) FROM test_fss_stream 16 | GROUP BY k 17 | """ 18 | desc = ('k', 'x') 19 | pipeline.create_cv('test_fss_agg', q) 20 | 21 | # items = range(14) 22 | #allocations = list(range(len(people))) 23 | items = list(range(14)) 24 | random.shuffle(items) 25 | a_items = items 26 | b_items = list(reversed(items)) 27 | 28 | values = list(map(lambda i: ('a', i), get_geometric_dist(a_items))) 29 | values.extend(list(map(lambda i: ('b', i), get_geometric_dist(b_items)))) 30 | random.shuffle(values) 31 | 32 | pipeline.insert('test_fss_stream', desc, values) 33 | result = list(pipeline.execute('SELECT k, topk_values(topk_agg) FROM test_fss_agg ORDER BY k')) 34 | topk = map(int, result[0][1].rstrip('}').lstrip('{').split(',')) 35 | assert sorted(topk) == sorted(a_items[-5:]) 36 | topk = map(int, result[1][1].rstrip('}').lstrip('{').split(',')) 37 | assert sorted(topk) == sorted(b_items[-5:]) 38 | -------------------------------------------------------------------------------- /include/combiner_receiver.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * combiner_receiver.h 4 | * An implementation of DestReceiver that that allows combiners to receive 5 | * tuples from worker processes. 6 | * 7 | * Copyright (c) 2023, Tantor Labs, Inc. 8 | * Copyright (c) 2018, PipelineDB, Inc. 9 | * 10 | * ------------------------------------------------------------------------- 11 | */ 12 | #ifndef COMBINER_RECEIVER_H 13 | #define COMBINER_RECEIVER_H 14 | 15 | #include "tcop/dest.h" 16 | #include "executor.h" 17 | 18 | typedef struct CombinerReceiver 19 | { 20 | BatchReceiver base; 21 | DestReceiver pub; 22 | ContQuery *cont_query; 23 | ContExecutor *cont_exec; 24 | FunctionCallInfo hash_fcinfo; 25 | FuncExpr *hashfn; 26 | 27 | uint64 name_hash; 28 | List **tups_per_combiner; 29 | } CombinerReceiver; 30 | 31 | typedef bool (*CombinerReceiveFunc) (ContQuery *query, uint32 shard_hash, uint64 group_hash, HeapTuple tup); 32 | extern CombinerReceiveFunc CombinerReceiveHook; 33 | typedef void (*CombinerFlushFunc) (void); 34 | extern CombinerFlushFunc CombinerFlushHook; 35 | 36 | extern BatchReceiver *CreateCombinerReceiver(ContExecutor *cont_exec, ContQuery *query, Tuplestorestate *buffer); 37 | extern void SetCombinerDestReceiverHashFunc(BatchReceiver *self, FuncExpr *hash); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /include/reader.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * reader.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef READER_H 11 | #define READER_H 12 | 13 | #include "postgres.h" 14 | 15 | #include "access/htup.h" 16 | #include "access/tupdesc.h" 17 | #include "nodes/bitmapset.h" 18 | 19 | #include "pzmq.h" 20 | 21 | typedef struct ipc_tuple 22 | { 23 | TupleDesc desc; 24 | List *record_descs; 25 | HeapTuple tup; 26 | uint64 hash; 27 | } ipc_tuple; 28 | 29 | typedef struct ipc_tuple_reader_batch 30 | { 31 | Bitmapset *queries; 32 | bool has_acks; 33 | List *sync_acks; /* only valid at when the iteration is complete */ 34 | List *flush_acks; 35 | int ntups; 36 | Size nbytes; 37 | } ipc_tuple_reader_batch; 38 | 39 | extern void ipc_tuple_reader_init(void); 40 | extern void ipc_tuple_reader_destroy(void); 41 | 42 | #define ipc_tuple_reader_poll(timeout) (pzmq_poll(timeout)) 43 | extern ipc_tuple_reader_batch *ipc_tuple_reader_pull(void); 44 | extern void ipc_tuple_reader_reset(void); 45 | extern void ipc_tuple_reader_ack(void); 46 | 47 | extern ipc_tuple *ipc_tuple_reader_next(Oid query_id); 48 | extern void ipc_tuple_reader_rewind(void); 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /src/test/regress/expected/stream_targets.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE test_stream_targets_stream (x int) SERVER pipelinedb; 2 | CREATE VIEW test_stream_targets0 AS SELECT COUNT(*) FROM test_stream_targets_stream; 3 | CREATE VIEW test_stream_targets1 AS SELECT COUNT(*) FROM test_stream_targets_stream; 4 | CREATE VIEW test_stream_targets2 AS SELECT COUNT(*) FROM test_stream_targets_stream; 5 | INSERT INTO test_stream_targets_stream (x) VALUES (1); 6 | SET SESSION pipelinedb.stream_targets TO "test_stream_targets1, test_stream_targets2"; 7 | INSERT INTO test_stream_targets_stream (x) VALUES (1); 8 | SET SESSION pipelinedb.stream_targets TO test_stream_targets2; 9 | INSERT INTO test_stream_targets_stream (x) VALUES (1); 10 | SET SESSION pipelinedb.stream_targets TO DEFAULT; 11 | INSERT INTO test_stream_targets_stream (x) VALUES (1); 12 | SELECT * FROM test_stream_targets0; 13 | count 14 | ------- 15 | 2 16 | (1 row) 17 | 18 | SELECT * FROM test_stream_targets1; 19 | count 20 | ------- 21 | 3 22 | (1 row) 23 | 24 | SELECT * FROM test_stream_targets2; 25 | count 26 | ------- 27 | 4 28 | (1 row) 29 | 30 | DROP FOREIGN TABLE test_stream_targets_stream CASCADE; 31 | NOTICE: drop cascades to 3 other objects 32 | DETAIL: drop cascades to view test_stream_targets0 33 | drop cascades to view test_stream_targets1 34 | drop cascades to view test_stream_targets2 35 | -------------------------------------------------------------------------------- /src/test/py/test_vacuum.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | from collections import namedtuple 3 | import getpass 4 | import os 5 | import psycopg2 6 | import psycopg2.extensions 7 | import random 8 | import threading 9 | import time 10 | 11 | 12 | def test_concurrent_vacuum_full(pipeline, clean_db): 13 | pipeline.create_stream('test_vacuum_stream', x='int') 14 | pipeline.create_cv( 15 | 'test_vacuum_full', 16 | 'SELECT x::int, COUNT(*) FROM test_vacuum_stream GROUP BY x') 17 | stop = False 18 | 19 | def insert(): 20 | while not stop: 21 | values = [(random.randint(0, 1000000),) for _ in range(1000)] 22 | pipeline.insert('test_vacuum_stream', ('x',), values) 23 | time.sleep(0.01) 24 | 25 | threads = [threading.Thread(target=insert) for _ in range(4)] 26 | map(lambda t: t.start(), threads) 27 | 28 | # Insert data for a little bit so we have enough work to do while 29 | # vacuuming. 30 | time.sleep(20) 31 | 32 | conn = psycopg2.connect('dbname=postgres user=%s host=localhost port=%s' % 33 | (getpass.getuser(), pipeline.port)) 34 | conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) 35 | cur = conn.cursor() 36 | cur.execute('VACUUM FULL test_vacuum_full') 37 | conn.close() 38 | 39 | # Now kill the insert threads. 40 | stop = True 41 | map(lambda t: t.join(), threads) 42 | -------------------------------------------------------------------------------- /pipelinedb--1.1.0--1.2.0.sql: -------------------------------------------------------------------------------- 1 | 2 | -- Add system metadata for partitions 3 | ALTER TABLE pipelinedb.cont_query ADD partition_duration int4; 4 | ALTER TABLE pipelinedb.cont_query ADD partition_key_attno int2; 5 | 6 | DROP AGGREGATE combine(anyelement); 7 | /* 8 | * Dummy combine aggregate for user combines 9 | * 10 | * User combines will always take a finalized aggregate value as input 11 | * and return a combined aggregate of the same type, so this dummy aggregate 12 | * ensures we make it through the analyzer with correct types everywhere. 13 | */ 14 | CREATE AGGREGATE combine(anyelement) ( 15 | sfunc = combine_trans_dummy, 16 | msfunc = combine_trans_dummy, 17 | minvfunc = combine_trans_dummy, 18 | mstype = anyelement, 19 | minitcond = 'combine_dummy', 20 | stype = anyelement, 21 | parallel = safe 22 | ); 23 | 24 | DROP AGGREGATE sw_combine(anyelement); 25 | /* 26 | * Dummy combine aggregate used in SW overlay queries 27 | * 28 | * It's convenient to differentiate between these and user combines in the planner, 29 | * so we indicate SW overlay combines using this aggregate 30 | */ 31 | CREATE AGGREGATE sw_combine(anyelement) ( 32 | sfunc = combine_trans_dummy, 33 | msfunc = combine_trans_dummy, 34 | minvfunc = combine_trans_dummy, 35 | mstype = anyelement, 36 | minitcond = 'sw_combine_dummy', 37 | stype = anyelement, 38 | parallel = safe 39 | ); 40 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_activate.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_activate_s (x int) SERVER pipelinedb; 2 | 3 | CREATE VIEW cont_activate_v0 AS SELECT count(*) FROM cont_activate_s; 4 | CREATE VIEW cont_activate_v1 AS SELECT count(*) FROM cont_activate_s; 5 | 6 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 7 | 8 | SELECT * FROM cont_activate_v0; 9 | SELECT * FROM cont_activate_v1; 10 | 11 | SELECT pipelinedb.deactivate('cont_activate_v0'); 12 | 13 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 14 | 15 | SELECT * FROM cont_activate_v0; 16 | SELECT * FROM cont_activate_v1; 17 | 18 | SELECT pipelinedb.deactivate('cont_activate_v1'); 19 | 20 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 21 | 22 | SELECT * FROM cont_activate_v0; 23 | SELECT * FROM cont_activate_v1; 24 | 25 | SELECT pipelinedb.activate('cont_activate_v0'); 26 | 27 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 28 | 29 | SELECT * FROM cont_activate_v0; 30 | SELECT * FROM cont_activate_v1; 31 | 32 | SELECT pipelinedb.activate('cont_activate_v1'); 33 | 34 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 35 | 36 | SELECT * FROM cont_activate_v0; 37 | SELECT * FROM cont_activate_v1; 38 | 39 | DROP VIEW cont_activate_v0; 40 | DROP VIEW cont_activate_v1; 41 | 42 | DROP FOREIGN TABLE cont_activate_s CASCADE; 43 | -------------------------------------------------------------------------------- /include/tdigest.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * tdigest.h 4 | * Interface for t-digest support 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef TDIGEST_H 12 | #define TDIGEST_H 13 | 14 | #include "c.h" 15 | #include "nodes/pg_list.h" 16 | 17 | typedef struct Centroid 18 | { 19 | uint64 weight; 20 | float8 mean; 21 | } Centroid; 22 | 23 | typedef struct TDigest { 24 | uint32 vl_len_; 25 | 26 | float8 compression; 27 | uint32 threshold; 28 | uint32 size; 29 | 30 | uint64 total_weight; 31 | float8 min; 32 | float8 max; 33 | 34 | List *unmerged_centroids; 35 | uint32 num_centroids; 36 | Centroid centroids[1]; 37 | } TDigest; 38 | 39 | extern TDigest *TDigestCreate(void); 40 | extern TDigest *TDigestCreateWithCompression(int compression); 41 | extern void TDigestDestroy(TDigest *t); 42 | extern TDigest *TDigestCopy(TDigest *t); 43 | 44 | extern TDigest *TDigestAdd(TDigest *t, float8 x, int64 w); 45 | extern TDigest *TDigestCompress(TDigest *t); 46 | extern TDigest *TDigestMerge(TDigest *t1, TDigest *t2); 47 | 48 | extern float8 TDigestCDF(TDigest *t, float8 x); 49 | extern float8 TDigestQuantile(TDigest *t, float8 q); 50 | 51 | extern Size TDigestSize(TDigest *t); 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /include/bloom.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * bloom.h 4 | * Interface for Bloom Filter support 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef PIPELINE_BLOOM_H 12 | #define PIPELINE_BLOOM_H 13 | 14 | #include "c.h" 15 | 16 | typedef struct BloomFilter 17 | { 18 | uint32 vl_len_; 19 | uint32_t m; 20 | uint16_t k; 21 | uint32_t blen; 22 | uint64_t b[1]; 23 | } BloomFilter; 24 | 25 | extern BloomFilter *BloomFilterCreateWithMAndK(uint32_t m, uint16_t k); 26 | extern BloomFilter *BloomFilterCreateWithPAndN(float8 p, uint32_t n); 27 | extern BloomFilter *BloomFilterCreate(void); 28 | extern void BloomFilterDestroy(BloomFilter *bf); 29 | 30 | extern BloomFilter *BloomFilterCopy(BloomFilter *bf); 31 | extern void BloomFilterAdd(BloomFilter *bf, void *key, Size size); 32 | extern bool BloomFilterContains(BloomFilter *bf, void *key, Size size); 33 | extern BloomFilter *BloomFilterUnion(BloomFilter *result, BloomFilter *incoming); 34 | extern BloomFilter *BloomFilterIntersection(BloomFilter *result, BloomFilter *incoming); 35 | extern uint64_t BloomFilterCardinality(BloomFilter *bf); 36 | extern float8 BloomFilterFillRatio(BloomFilter *bf); 37 | extern Size BloomFilterSize(BloomFilter *bf); 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /include/planner.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * planner.h 4 | * Interface for generating/modifying CQ plans 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | */ 10 | #ifndef CONT_PLAN_H 11 | #define CONT_PLAN_H 12 | 13 | #include "executor/execdesc.h" 14 | #include "optimizer/planner.h" 15 | #include "pipeline_query.h" 16 | #include "scheduler.h" 17 | 18 | extern bool debug_simulate_continuous_query; 19 | 20 | extern void InstallPlannerHooks(void); 21 | 22 | extern PlannedStmt *GetContPlan(ContQuery *view, ContQueryProcType type); 23 | extern PlannedStmt *GetGroupsLookupPlan(Query *query, ParseState *pstate); 24 | extern void SetCombinerPlanTuplestorestate(PlannedStmt *plan, Tuplestorestate *tupstore); 25 | extern FuncExpr *GetGroupHashIndexExpr(ResultRelInfo *ri); 26 | extern PlannedStmt *GetCombinerLookupPlan(ContQuery *view); 27 | extern PlannedStmt *GetContViewOverlayPlan(ContQuery *view); 28 | 29 | extern FuncExpr *GetGroupHashIndexExpr(ResultRelInfo *ri); 30 | 31 | extern EState *CreateEState(QueryDesc *query_desc); 32 | extern void SetEStateSnapshot(EState *estate); 33 | extern void UnsetEStateSnapshot(EState *estate); 34 | 35 | extern PlannedStmt *PipelinePlanner(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams); 36 | 37 | extern void end_plan(QueryDesc *query_desc); 38 | #endif 39 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_part_fillfactor.sql: -------------------------------------------------------------------------------- 1 | drop function if exists list_partitions(name); 2 | create function 3 | list_partitions( 4 | baserel name 5 | ) 6 | returns table ( 7 | parent name, child text, options text[] 8 | ) 9 | as $BODY$ 10 | begin 11 | return query select 12 | parent.relname as parent, 13 | regexp_replace(child.relname, 'mrel_[0-9]+_', 'mrel_OID_') as child, 14 | child.reloptions as options 15 | from pg_inherits 16 | join pg_class parent on pg_inherits.inhparent = parent.oid 17 | join pg_class child on pg_inherits.inhrelid = child.oid 18 | join pg_namespace nmsp_parent on nmsp_parent.oid = parent.relnamespace 19 | join pg_namespace nmsp_child on nmsp_child.oid = child.relnamespace 20 | where 21 | parent.relname = baserel || '_mrel' 22 | order by child 23 | ; 24 | end; 25 | $BODY$ 26 | language plpgsql; 27 | 28 | create foreign table part_ff_stream (ts timestamp, x int) server pipelinedb; 29 | 30 | create view cv_part_ff 31 | with (partition_by=ts, partition_duration='1 hour') 32 | as select ts, count(*) from part_ff_stream group by ts; 33 | 34 | \d+ cv_part_ff_mrel; 35 | 36 | insert into part_ff_stream values ('2024-11-12 10:18:42', 1); 37 | 38 | select * from list_partitions('cv_part_ff'); 39 | 40 | insert into part_ff_stream values ('2024-11-12 20:18:42', 2); 41 | 42 | select * from list_partitions('cv_part_ff'); 43 | 44 | drop foreign table part_ff_stream cascade; 45 | -------------------------------------------------------------------------------- /src/test/py/test_distinct.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | from collections import defaultdict 3 | import random 4 | 5 | 6 | def test_distinct(pipeline, clean_db): 7 | """ 8 | Verify that streaming SELECT DISTINCT ON (...) works 9 | """ 10 | pipeline.create_stream('stream0', x='int', y='int', z='int') 11 | pipeline.create_table('table0', x='int', y='int', z='int') 12 | q = 'SELECT DISTINCT ON (x::int, y::int - z::int) x::int, y::int FROM stream0' 13 | pipeline.create_cv('test_distinct', q) 14 | 15 | uniques = defaultdict(set) 16 | values = [] 17 | for _ in range(2000): 18 | x, y, z = random.randint(0, 20), random.randint(0, 20), random.randint(0, 20) 19 | values.append((x, y, z)) 20 | uniques[(x, y - z)].add(y) 21 | 22 | pipeline.insert('stream0', ['x', 'y', 'z'], values) 23 | pipeline.insert('table0', ['x', 'y', 'z'], values) 24 | 25 | q = """ 26 | SELECT DISTINCT ON (x::int, y::int - z::int) x::int, y::int FROM table0 27 | """ 28 | expected = pipeline.execute(q) 29 | expected = len(expected) 30 | 31 | assert expected < 2000 32 | 33 | result = pipeline.execute('SELECT COUNT(*) FROM test_distinct')[0] 34 | 35 | assert expected == result['count'] 36 | 37 | # Check if the first row was selected for uniques 38 | result = pipeline.execute('SELECT * FROM test_distinct') 39 | reverse_uniques = defaultdict(set) 40 | 41 | for (x, _), ys in uniques.items(): 42 | for y in ys: 43 | reverse_uniques[y].add(x) 44 | 45 | for row in result: 46 | assert row['x'] in reverse_uniques[row['y']] 47 | -------------------------------------------------------------------------------- /src/test/regress/sql/hash_group.sql: -------------------------------------------------------------------------------- 1 | SELECT pipelinedb.hash_group(0::int, 0::int, 0::int, 0::int); 2 | SELECT pipelinedb.hash_group('0'::text, '2015-02-01'::timestamp, 1.2); 3 | SELECT pipelinedb.hash_group(null::text); 4 | SELECT pipelinedb.hash_group(null::text, null::text); 5 | 6 | SELECT pipelinedb.hash_group(1::int8, 2::int4); 7 | SELECT pipelinedb.hash_group(1::int4, 2::int8); 8 | SELECT pipelinedb.hash_group(0::int2, null::int2); 9 | SELECT pipelinedb.hash_group(null::int2, null::int2); 10 | 11 | SELECT date_trunc('second', '2015-01-01'::timestamp) + i * interval '1 second' AS ts, pipelinedb.ls_hash_group(date_trunc('second', '2015-01-01'::timestamp) + i * interval '1 second', 0::int4) AS key 12 | FROM generate_series(1, 100) AS i ORDER BY key; 13 | 14 | -- Ensure that hash index is created and cannot be dropped 15 | CREATE FOREIGN TABLE hash_group_stream (x int, y timestamptz) SERVER pipelinedb; 16 | CREATE VIEW hash_group AS SELECT x::int, COUNT(*) FROM hash_group_stream GROUP BY x; 17 | CREATE VIEW ls_hash_group1 AS SELECT x::int, minute(y::timestamptz), COUNT(*) FROM hash_group_stream WHERE ( arrival_timestamp > clock_timestamp() - interval '5 hour' ) GROUP BY x, minute; 18 | CREATE VIEW ls_hash_group2 AS SELECT x::int, y::timestamptz, COUNT(*) FROM hash_group_stream GROUP BY x, y; 19 | 20 | \d+ hash_group_mrel; 21 | \d+ ls_hash_group1_mrel; 22 | \d+ ls_hash_group2_mrel; 23 | 24 | DROP INDEX hash_group_mrel_expr_idx; 25 | DROP INDEX ls_hash_group1_mrel_expr_idx; 26 | DROP INDEX ls_hash_group2_mrel_expr_idx; 27 | 28 | DROP FOREIGN TABLE hash_group_stream CASCADE; 29 | -------------------------------------------------------------------------------- /src/test/regress/sql/pipeline_stream.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE stream0 (id integer) SERVER pipelinedb; 2 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 3 | CREATE VIEW ps0 AS SELECT id FROM stream0; 4 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 5 | CREATE VIEW ps1 AS SELECT count(*) FROM stream0; 6 | CREATE VIEW ps2 AS SELECT id FROM stream0; 7 | CREATE FOREIGN TABLE stream1 (x integer, y timestamp) SERVER pipelinedb; 8 | CREATE VIEW ps3 AS SELECT x, y FROM stream1; 9 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 10 | CREATE VIEW ps4 AS SELECT id::text FROM stream0; 11 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 12 | CREATE FOREIGN TABLE stream2 (x integer) SERVER pipelinedb; 13 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 14 | CREATE VIEW ps5 AS SELECT x FROM stream2; 15 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 16 | DROP VIEW ps0; 17 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 18 | DROP VIEW ps1; 19 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 20 | DROP VIEW ps2; 21 | DROP VIEW ps3; 22 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 23 | DROP VIEW ps5; 24 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 25 | DROP FOREIGN TABLE stream2; 26 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 27 | DROP FOREIGN TABLE stream1; 28 | DROP FOREIGN TABLE stream0 CASCADE; 29 | SELECT schema, name, queries FROM pipelinedb.get_streams() ORDER BY name; 30 | -------------------------------------------------------------------------------- /src/test/regress/serial_schedule: -------------------------------------------------------------------------------- 1 | test: pipeline_stream 2 | test: cont_limit 3 | test: cont_transform 4 | test: cont_view_sanity 5 | test: cont_sum 6 | test: cont_avg 7 | test: cont_count 8 | test: stream_targets 9 | test: date_round 10 | test: stream_insert_subselect 11 | test: stream_exprs 12 | test: prepared_stream_insert 13 | test: cont_view_namespace 14 | test: cont_bool_agg 15 | test: cont_min_max 16 | test: cont_regr 17 | test: cont_pk 18 | test: matrel_constraints 19 | test: cont_json_agg 20 | test: user_combine 21 | test: cont_sw_avg 22 | test: cont_stats_agg 23 | test: cont_array_agg 24 | test: cont_set_agg 25 | test: cont_distinct 26 | test: cont_hll_agg 27 | test: hll 28 | test: cont_hs_agg 29 | test: cont_os_agg 30 | test: typed_streams 31 | test: bloom 32 | test: cont_complex_types 33 | test: topk 34 | test: cont_topk_agg 35 | test: dist 36 | test: cont_dist_agg 37 | test: freq 38 | test: cont_freq_agg 39 | test: keyed_min_max 40 | test: first_values 41 | test: bucket_agg 42 | test: output_streams 43 | test: delta_streams 44 | test: stream_table_join 45 | test: cont_activate 46 | test: cont_subselect 47 | test: hash_group 48 | test: part_ttl 49 | test: ttl_expiration 50 | test: sw_expiration 51 | test: analyze_cont_view 52 | test: cont_stats 53 | test: cont_sw_sum 54 | test: cont_sw_bool_agg 55 | test: cont_sw_min_max 56 | test: cont_sw_regr 57 | test: cont_sw_stats 58 | test: cont_sw_count 59 | test: cont_object_agg 60 | test: cont_sw_hs_agg 61 | test: cont_sw_os_agg 62 | test: create_cont_view 63 | test: cont_alter 64 | test: cont_sw_object_agg 65 | test: pipeline_regress 66 | test: cont_grouping_sets 67 | test: cont_matrel 68 | test: cont_index 69 | test: cont_part_basic 70 | test: cont_part_regress 71 | -------------------------------------------------------------------------------- /src/test/regress/sql/bloom.sql: -------------------------------------------------------------------------------- 1 | SELECT bloom_contains((SELECT bloom_agg(g) FROM generate_series(1, 100) AS g), x) FROM generate_series(1, 110) AS x; 2 | 3 | SELECT bloom_contains( 4 | (SELECT bloom_union_agg(bf) FROM 5 | (SELECT bloom_agg(g) AS bf FROM generate_series(1, 100) AS g 6 | UNION ALL 7 | SELECT bloom_agg(g) AS bf FROM generate_series(101, 200) AS g) _), x) 8 | FROM generate_series(1, 210) AS x; 9 | 10 | SELECT bloom_cardinality( 11 | bloom_union( 12 | (SELECT bloom_agg(x) FROM generate_series(1, 1000) AS x), 13 | (SELECT bloom_agg(x) FROM generate_series(100, 1100) AS x), 14 | NULL)); 15 | 16 | -- Different BF dimension 17 | SELECT bloom_cardinality( 18 | bloom_union( 19 | (SELECT bloom_agg(x, 0.5, 1000) FROM generate_series(1, 1000) AS x), 20 | (SELECT bloom_agg(x) FROM generate_series(100, 1100) AS x))); 21 | 22 | -- Wrong type 23 | SELECT bloom_cardinality( 24 | bloom_union( 25 | 'not a bloom filter', 26 | (SELECT bloom_agg(x) FROM generate_series(100, 1100) AS x))); 27 | 28 | SELECT bloom_cardinality( 29 | bloom_intersection( 30 | (SELECT bloom_agg(x) FROM generate_series(1, 1000) AS x), 31 | (SELECT bloom_agg(x) FROM generate_series(100, 1100) AS x), 32 | NULL)); 33 | 34 | -- Different BF dimension 35 | SELECT bloom_cardinality( 36 | bloom_intersection( 37 | (SELECT bloom_agg(x, 0.5, 1000) FROM generate_series(1, 1000) AS x), 38 | (SELECT bloom_agg(x) FROM generate_series(100, 1100) AS x))); 39 | 40 | -- Wrong type 41 | SELECT bloom_cardinality( 42 | bloom_intersection( 43 | 'not a bloom filter', 44 | (SELECT bloom_agg(x) FROM generate_series(100, 1100) AS x))); 45 | -------------------------------------------------------------------------------- /src/test/regress/sql/prepared_stream_insert.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE prep_insert_stream (x float8, y int, z int) SERVER pipelinedb; 2 | 3 | CREATE VIEW prep_insert0 AS SELECT COUNT(*) FROM prep_insert_stream; 4 | CREATE VIEW prep_insert1 AS SELECT sum(x::float8) AS fsum, sum(y::int8) AS isum FROM prep_insert_stream; 5 | CREATE VIEW prep_insert2 AS SELECT sum(x::integer) AS isum, sum(y::int4) AS i4sum FROM prep_insert_stream; 6 | 7 | CREATE TABLE prep_insert_t (x integer, y integer, z integer); 8 | 9 | PREPARE prep0 AS INSERT INTO prep_insert_stream (x) VALUES ($1); 10 | PREPARE prep1 AS INSERT INTO prep_insert_stream (x, y) VALUES ($1, $2); 11 | PREPARE prep_t AS INSERT INTO prep_insert_t (x, y, z) VALUES ($1, $2, $3); 12 | 13 | EXECUTE prep0(1); 14 | EXECUTE prep0(1); 15 | EXECUTE prep0(1); 16 | EXECUTE prep0(1); 17 | EXECUTE prep0(1); 18 | EXECUTE prep0(1); 19 | EXECUTE prep0(1); 20 | EXECUTE prep0(1.5); 21 | EXECUTE prep0(1.5); 22 | EXECUTE prep0(1.5); 23 | EXECUTE prep0(1.5); 24 | EXECUTE prep0(1.5); 25 | EXECUTE prep0(1.5); 26 | EXECUTE prep0(1.5); 27 | 28 | EXECUTE prep1(1, 1.1); 29 | EXECUTE prep1(1, 1.1); 30 | EXECUTE prep1(1, 1.1); 31 | EXECUTE prep1(1, 1.1); 32 | EXECUTE prep1(1, 1.1); 33 | EXECUTE prep1(1, 1.1); 34 | EXECUTE prep1(1, 1.1); 35 | EXECUTE prep1(1, 1.1); 36 | EXECUTE prep1(1, 1.1); 37 | EXECUTE prep1(1, 1.1); 38 | 39 | EXECUTE prep_t(0, 0, 0); 40 | EXECUTE prep_t(0, 0, 0); 41 | EXECUTE prep_t(0, 0, 0); 42 | EXECUTE prep_t(0, 0, 0); 43 | EXECUTE prep_t(0, 0, 0); 44 | EXECUTE prep_t(0, 0, 0); 45 | EXECUTE prep_t(0, 0, 0); 46 | EXECUTE prep_t(0, 0, 0); 47 | 48 | SELECT * FROM prep_insert0; 49 | SELECT * FROM prep_insert1; 50 | SELECT * FROM prep_insert2; 51 | 52 | DROP FOREIGN TABLE prep_insert_stream CASCADE; 53 | DROP TABLE prep_insert_t; 54 | -------------------------------------------------------------------------------- /include/compat.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * compat.h 4 | * Interface for functionality that breaks between major PG versions 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #include "postgres.h" 12 | 13 | #include "catalog/objectaddress.h" 14 | #include "executor/tuptable.h" 15 | #include "nodes/execnodes.h" 16 | #include "optimizer/pathnode.h" 17 | 18 | extern bool CompatProcOidIsAgg(Oid oid); 19 | extern void CompatAnalyzeVacuumStmt(VacuumStmt *stmt); 20 | 21 | extern Relids CompatCalcNestLoopRequiredOuter(Path *outer, Path *inner); 22 | extern void CompatPrepareEState(PlannedStmt *pstmt, EState *estate); 23 | 24 | extern void 25 | CompatExecTuplesHashPrepare(int numCols,Oid *eqOperators, 26 | #if (PG_VERSION_NUM < 110000) 27 | FmgrInfo **eqFunctions, 28 | #else 29 | Oid **eqFunctions, 30 | #endif 31 | FmgrInfo **hashFunctions); 32 | 33 | extern TupleHashTable 34 | CompatBuildTupleHashTable(TupleDesc inputDesc, 35 | int numCols, AttrNumber *keyColIdx, 36 | #if (PG_VERSION_NUM < 110000) 37 | FmgrInfo *eqfuncs, 38 | #else 39 | Oid *eqfuncs, 40 | #endif 41 | FmgrInfo *hashfunctions, 42 | Oid *collations, 43 | long nbuckets, Size additionalsize, 44 | MemoryContext tablecxt, 45 | MemoryContext tempcxt, bool use_variable_hash_iv); 46 | 47 | extern char *CompatGetAttName(Oid relid, AttrNumber att); 48 | extern void ComaptExecAssignResultTypeFromTL(PlanState *ps); 49 | extern HeapTuple CompatSearchSysCacheLocked1(int cacheId, Datum key1); 50 | extern void CompatUnlockTupleInplaceUpdate(Relation relation, ItemPointer tid); 51 | -------------------------------------------------------------------------------- /src/test/py/test_tablespace.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import glob 3 | import os 4 | import shutil 5 | 6 | 7 | def test_tablespace(pipeline, clean_db): 8 | """ 9 | Verify that CVs can be created within tablespaces 10 | """ 11 | path = os.path.abspath('test_tablespace') 12 | if os.path.exists(path): 13 | shutil.rmtree(path) 14 | 15 | os.mkdir(path) 16 | pipeline.execute("CREATE TABLESPACE test_tablespace LOCATION '%s'" % path) 17 | 18 | pipeline.create_stream('test_tablespace_s', x='int') 19 | 20 | q = 'SELECT x % 10 AS g, count(DISTINCT x) FROM test_tablespace_s GROUP BY g' 21 | pipeline.create_cv('test_tablespace0', q) 22 | pipeline.create_cv('test_tablespace1', q, tablespace='test_tablespace') 23 | pipeline.insert('test_tablespace_s', ('x',), [(x,) for x in range(10000)]) 24 | 25 | result0 = pipeline.execute('SELECT count(*) FROM test_tablespace0') 26 | result1 = pipeline.execute('SELECT count(*) FROM test_tablespace1') 27 | 28 | assert len(result0) == 1 29 | assert len(result1) == 1 30 | assert result0[0]['count'] == result1[0]['count'] 31 | 32 | result0 = pipeline.execute('SELECT combine(count) FROM test_tablespace0') 33 | result1 = pipeline.execute('SELECT combine(count) FROM test_tablespace1') 34 | 35 | assert len(result0) == 1 36 | assert len(result1) == 1 37 | assert result0[0]['combine'] == result1[0]['combine'] 38 | 39 | # Now verify that test_tablespace1 is physically in the tablespace 40 | row = pipeline.execute("SELECT oid FROM pg_class WHERE relname = 'test_tablespace1_mrel'") 41 | oid = row[0]['oid'] 42 | 43 | found = glob.glob('test_tablespace/*/*/%d' % oid) 44 | assert len(found) == 1 45 | 46 | pipeline.drop_all() 47 | pipeline.execute('DROP TABLESPACE test_tablespace') 48 | shutil.rmtree(path) -------------------------------------------------------------------------------- /src/test/py/test_os_aggs.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import random 3 | 4 | def test_percentile_cont_agg(pipeline, clean_db): 5 | range_top = 100000 6 | q = [0.0, 0.01, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.99, 1.0] 7 | 8 | batches = [] 9 | min_seen = range_top 10 | max_seen = 0 11 | for _ in range(10): 12 | b = [(random.randint(0, range_top),) for _ in range(5000)] 13 | min_seen = min(min_seen, min(b)[0]) 14 | max_seen = max(max_seen, max(b)[0]) 15 | batches.append(b) 16 | 17 | pipeline.create_stream('test_stream', x='int') 18 | query = '''SELECT 19 | percentile_cont(ARRAY[%s]) 20 | WITHIN GROUP (ORDER BY x::integer) FROM %s 21 | ''' % (', '.join(map(lambda f: str(f), q)), '%s') 22 | 23 | pipeline.create_cv('test_cq_percentile_cont', query % 'test_stream') 24 | pipeline.create_table('test_percentile_cont', x='integer') 25 | 26 | for b in batches: 27 | pipeline.insert('test_stream', ('x',), b) 28 | pipeline.insert('test_percentile_cont', ('x',), b) 29 | 30 | actual = pipeline.execute(query % 'test_percentile_cont') 31 | result = pipeline.execute('SELECT * FROM test_cq_percentile_cont') 32 | 33 | actual = actual[0]['percentile_cont'] 34 | result = result[0]['percentile_cont'] 35 | 36 | assert len(actual) == len(result) 37 | assert result == sorted(result) 38 | diff = [abs(actual[i] - result[i]) for i in range(len(actual))] 39 | 40 | # 0th and 100th percentile should be accurate. 41 | assert result[0] == min_seen 42 | assert result[-1] == max_seen 43 | 44 | # 1st and 99th percentile should be within 0.1%. 45 | assert diff[1] <= 0.001 * range_top 46 | assert diff[-2] <= 0.001 * range_top 47 | 48 | # All percentiles should be within 0.5%. 49 | assert all(x <= 0.005 * range_top for x in diff) 50 | -------------------------------------------------------------------------------- /include/cmsketch.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * cmsketch.h 4 | * Interface for Count-Min Sketch support 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef PIPELINE_CMSKETCH_H 12 | #define PIPELINE_CMSKETCH_H 13 | 14 | #include "c.h" 15 | 16 | /* 17 | * Unfortunately we can't do top-K with Count-Min Sketch for continuous 18 | * queries because it will require us to store O(n) values. This isn't a 19 | * problem in the case when Count-Min Sketch structures are not being 20 | * merged in which case only O(1) values need to be stored. 21 | */ 22 | typedef struct CountMinSketch 23 | { 24 | uint32 vl_len_; 25 | uint64_t count; 26 | uint32_t d; 27 | uint32_t w; 28 | uint32_t table[1]; 29 | } CountMinSketch; 30 | 31 | extern CountMinSketch *CountMinSketchCreateWithDAndW(uint32_t d, uint32_t w); 32 | extern CountMinSketch *CountMinSketchCreateWithEpsAndP(float8 epsilon, float8 p); 33 | extern CountMinSketch *CountMinSketchCreate(void); 34 | extern void CountMinSketchDestroy(CountMinSketch *cms); 35 | 36 | extern CountMinSketch *CountMinSketchCopy(CountMinSketch *cms); 37 | extern void CountMinSketchAdd(CountMinSketch *cms, void *key, Size size, uint32_t count); 38 | extern uint32_t CountMinSketchEstimateFrequency(CountMinSketch *cms, void *key, Size size); 39 | extern float8 CountMinSketchEstimateNormFrequency(CountMinSketch *cms, void *key, Size size); 40 | extern uint64_t CountMinSketchTotal(CountMinSketch *cms); 41 | extern CountMinSketch *CountMinSketchMerge(CountMinSketch *result, CountMinSketch* incoming); 42 | extern Size CountMinSketchSize(CountMinSketch *cms); 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_freq_agg.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_freq_agg_stream (x integer, y text) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_freq_agg0 AS 4 | SELECT 5 | x % 10 AS g, 6 | freq_agg(x) AS x_freq, 7 | freq_agg(y) AS y_freq 8 | FROM cont_freq_agg_stream 9 | GROUP BY g; 10 | 11 | INSERT INTO cont_freq_agg_stream (x, y) SELECT x, x::text FROM generate_series(1, 1000) x; 12 | INSERT INTO cont_freq_agg_stream (x, y) SELECT 42, '42'::text FROM generate_series(1, 1000) x; 13 | INSERT INTO cont_freq_agg_stream (x, y) SELECT 0, '00'::text FROM generate_series(1, 100) x; 14 | INSERT INTO cont_freq_agg_stream (x, y) SELECT 1, '01'::text FROM generate_series(1, 100) x; 15 | INSERT INTO cont_freq_agg_stream (x, y) SELECT 2, '02'::text FROM generate_series(1, 100) x; 16 | INSERT INTO cont_freq_agg_stream (x, y) SELECT 3, '03'::text FROM generate_series(1, 100) x; 17 | 18 | SELECT g, freq(x_freq, 42) FROM test_freq_agg0 ORDER BY g; 19 | SELECT g, freq(y_freq, '42') FROM test_freq_agg0 ORDER BY g; 20 | 21 | SELECT g, freq(x_freq, 0) FROM test_freq_agg0 ORDER BY g; 22 | SELECT g, freq(y_freq, '00') FROM test_freq_agg0 ORDER BY g; 23 | 24 | SELECT g, freq(x_freq, 1) FROM test_freq_agg0 ORDER BY g; 25 | SELECT g, freq(y_freq, '01') FROM test_freq_agg0 ORDER BY g; 26 | 27 | SELECT g, freq(x_freq, 2) FROM test_freq_agg0 ORDER BY g; 28 | SELECT g, freq(y_freq, '02') FROM test_freq_agg0 ORDER BY g; 29 | 30 | SELECT g, freq(x_freq, 3) FROM test_freq_agg0 ORDER BY g; 31 | SELECT g, freq(y_freq, '03') FROM test_freq_agg0 ORDER BY g; 32 | 33 | INSERT INTO cont_freq_agg_stream (x, y) SELECT 42, '42'::text FROM generate_series(1, 1000) x; 34 | 35 | SELECT g, freq(x_freq, 42) FROM test_freq_agg0 ORDER BY g; 36 | SELECT g, freq(y_freq, '42') FROM test_freq_agg0 ORDER BY g; 37 | 38 | DROP FOREIGN TABLE cont_freq_agg_stream CASCADE; 39 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * whitespace=space-before-tab,trailing-space 2 | *.[chly] whitespace=space-before-tab,trailing-space,indent-with-non-tab,tabwidth=4 3 | *.dsl whitespace=space-before-tab,trailing-space,tab-in-indent 4 | *.patch -whitespace 5 | *.pl whitespace=space-before-tab,trailing-space,tabwidth=4 6 | *.po whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eof 7 | *.sgml whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eol 8 | *.x[ms]l whitespace=space-before-tab,trailing-space,tab-in-indent 9 | 10 | # Avoid confusing ASCII underlines with leftover merge conflict markers 11 | README conflict-marker-size=32 12 | README.* conflict-marker-size=32 13 | 14 | # Certain data files that contain special whitespace, and other special cases 15 | *.data -whitespace 16 | contrib/tsearch2/sql/tsearch2.sql whitespace=space-before-tab,blank-at-eof,-blank-at-eol 17 | doc/bug.template whitespace=space-before-tab,-blank-at-eof,blank-at-eol 18 | src/backend/catalog/sql_features.txt whitespace=space-before-tab,blank-at-eof,-blank-at-eol 19 | src/backend/tsearch/hunspell_sample.affix whitespace=-blank-at-eof 20 | 21 | # Test output files that contain extra whitespace 22 | *.out -whitespace 23 | contrib/*/output/*.source -whitespace 24 | src/test/regress/output/*.source -whitespace 25 | src/interfaces/ecpg/test/expected/* -whitespace 26 | src/interfaces/libpq/test/expected.out whitespace=-blank-at-eof 27 | 28 | # These files are maintained or generated elsewhere. We take them as is. 29 | configure -whitespace 30 | ppport.h -whitespace 31 | src/backend/regex/COPYRIGHT -whitespace 32 | src/backend/regex/re_syntax.n -whitespace 33 | src/backend/snowball/libstemmer/*.c -whitespace 34 | src/backend/utils/mb/Unicode/*-std.txt -whitespace 35 | src/include/snowball/libstemmer/* -whitespace 36 | src/timezone/data/* -whitespace 37 | -------------------------------------------------------------------------------- /src/test/py/test_stream_buffer.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import threading 3 | 4 | 5 | def test_stream_buffer(pipeline, clean_db): 6 | """ 7 | Run CQs which take different times to process an event in the stream buffer 8 | and each stream is read by a disjoint set of CQs. 9 | """ 10 | pipeline.create_stream('stream1', x='int', string='text') 11 | pipeline.create_stream('stream2', x='int', string='text') 12 | pipeline.create_cv('test_sbuf_1', 13 | 'SELECT x::int FROM stream1') 14 | pipeline.create_cv('test_sbuf_2', 15 | 'SELECT x::int, cq_sleep(0.002) FROM stream1') 16 | pipeline.create_cv('test_sbuf_3', 17 | 'SELECT x::int FROM stream2') 18 | pipeline.create_cv('test_sbuf_4', 19 | 'SELECT x::int, cq_sleep(0.002) FROM stream2') 20 | 21 | num_per_batch = 5000 22 | num_batches = 6 23 | # We're sending this 2k string with each event just to consume 24 | # more space in the stream buffer. That way we get to test the 25 | # wrapping of the buffer as well. 26 | values = [(i, 'a' * 2048) for i in range(num_per_batch)] 27 | 28 | def insert(stream): 29 | for _ in range(num_batches - 1): 30 | pipeline.insert(stream, ('x', 'string'), values) 31 | 32 | threads = [threading.Thread(target=insert, args=('stream1', )), 33 | threading.Thread(target=insert, args=('stream2', ))] 34 | 35 | [t.start() for t in threads] 36 | [t.join() for t in threads] 37 | 38 | pipeline.insert('stream1', ('x', 'string'), values) 39 | pipeline.insert('stream2', ('x', 'string'), values) 40 | 41 | q = 'SELECT COUNT(*) FROM test_sbuf_%d' 42 | r1 = pipeline.execute(q % 1)[0][0] 43 | r2 = pipeline.execute(q % 2)[0][0] 44 | r3 = pipeline.execute(q % 3)[0][0] 45 | r4 = pipeline.execute(q % 4)[0][0] 46 | 47 | assert r1 == r2 == r3 == r4 == num_batches * num_per_batch 48 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_sw_count.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cqswcount_stream (k text) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_count AS SELECT k::text, COUNT(*) FROM cqswcount_stream WHERE arrival_timestamp > clock_timestamp() - interval '60 second' GROUP BY k; 4 | 5 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'); 6 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'), ('y'), ('y'), ('y'), ('y'), ('y'), ('y'); 7 | 8 | SELECT * FROM test_count ORDER BY k; 9 | 10 | SELECT pg_sleep(1); 11 | 12 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'); 13 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'), ('y'), ('y'), ('y'), ('y'), ('y'), ('y'); 14 | 15 | SELECT * FROM test_count ORDER BY k; 16 | 17 | DROP VIEW test_count; 18 | 19 | CREATE VIEW sw_count0 AS SELECT COUNT(*) FROM cqswcount_stream WHERE arrival_timestamp > clock_timestamp() - interval '10 second'; 20 | 21 | CREATE VIEW sw_count1 AS SELECT combine(count) FROM sw_count0_mrel WHERE arrival_timestamp > clock_timestamp() - interval '5 second'; 22 | 23 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'); 24 | 25 | SELECT * FROM sw_count0; 26 | SELECT * FROM sw_count1; 27 | 28 | SELECT pg_sleep(6); 29 | 30 | SELECT * FROM sw_count0; 31 | SELECT * FROM sw_count1; 32 | 33 | DROP FOREIGN TABLE cqswcount_stream CASCADE; 34 | 35 | -- Verify that we can specify our own sliding-window column 36 | CREATE FOREIGN TABLE sw_col_s (x integer, ts timestamptz) SERVER pipelinedb; 37 | 38 | CREATE VIEW sw_col0 WITH (sw = '10 seconds', sw_column = 'ts') AS SELECT COUNT(*) FROM sw_col_s; 39 | 40 | INSERT INTO sw_col_s (x, ts) VALUES (0, now() - interval '8 seconds'); 41 | 42 | SELECT * FROM sw_col0; 43 | 44 | SELECT pg_sleep(3); 45 | 46 | SELECT * FROM sw_col0; 47 | 48 | DROP FOREIGN TABLE sw_col_s CASCADE; -------------------------------------------------------------------------------- /src/test/regress/sql/stream_exprs.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE test_exprs_stream (b boolean, t text, n numeric) SERVER pipelinedb; 2 | CREATE VIEW test_stream_exprs AS SELECT b::boolean, t::text, n::numeric FROM test_exprs_stream; 3 | 4 | INSERT INTO test_exprs_stream (b, t, n) VALUES (true and true, substring('string!', 1, 3), 1.2 + 100.0001); 5 | INSERT INTO test_exprs_stream (b, t, n) VALUES (1 < 2, 'first' || 'second', 100 % 2 * log(2, 3)); 6 | INSERT INTO test_exprs_stream (b, t, n) VALUES (1 is null, lower('UPPER'), pow(factorial(6)::float8 + 4, 3)); 7 | INSERT INTO test_exprs_stream (b, t, n) VALUES (true and true and true and false, trim(both 'x' from 'xTomxx'), pi() * mod(1 + 3, 4 * 8)); 8 | INSERT INTO test_exprs_stream (b, t, n) VALUES ('t', overlay('Txxxxas' placing 'hom' from 2 for 4), acos(1) * trunc(4.003)); 9 | INSERT INTO test_exprs_stream (b, t, n) VALUES ('on' and true, substring(upper('lower'), 1, 2), 42.82 - @(-12)); 10 | INSERT INTO test_exprs_stream (b, t, n) VALUES ('on' or 'off', btrim('xyxtrimyyx', 'xy'), 2^32); 11 | INSERT INTO test_exprs_stream (b, t, n) VALUES (1 * 2 > 2 * 3, md5(md5(md5('md5 me three times!'))), |/(64)); 12 | INSERT INTO test_exprs_stream (b, t, n) VALUES (1 + 4 * 3 - 14 < 1.2, 'true', pow(2::float8, 32::numeric)); 13 | INSERT INTO test_exprs_stream (b, t, n) VALUES (true and not false or true, substring('left', 1, 2) || substring('right', 1, 2), 1 << 4 << 4); 14 | INSERT INTO test_exprs_stream (b, t, n) VALUES (null and true, substring('x' || 'blah' || 'x', 2, 4), (100 + 24) * (16 * 3 + 4) - (16 << 2 >> 2)); 15 | 16 | -- Nonexistent fields should default to null 17 | INSERT INTO test_exprs_stream (b) VALUES (false); 18 | INSERT INTO test_exprs_stream (t) VALUES ('text!'); 19 | INSERT INTO test_exprs_stream (n) VALUES (1 - 1); 20 | 21 | SELECT * FROM test_stream_exprs ORDER BY b, t, n; 22 | 23 | DROP FOREIGN TABLE test_exprs_stream CASCADE; 24 | -------------------------------------------------------------------------------- /include/pipeline_stream.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * pipeline_stream.h 4 | * Interface for pipelinedb.stream catalog 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | * ------------------------------------------------------------------------- 10 | */ 11 | #ifndef PIPELINE_STREAM_H 12 | #define PIPELINE_STREAM_H 13 | 14 | #include "postgres.h" 15 | 16 | #include "nodes/parsenodes.h" 17 | #include "storage/lockdefs.h" 18 | #include "utils/relcache.h" 19 | 20 | #define PIPELINEDB_SERVER "pipelinedb" 21 | #define ARRIVAL_TIMESTAMP "arrival_timestamp" 22 | 23 | extern Oid PipelineStreamRelationOid; 24 | 25 | extern char *stream_targets; 26 | 27 | typedef struct FormData_pipeline_stream 28 | { 29 | Oid relid; 30 | #ifdef CATALOG_VARLEN 31 | bytea queries; 32 | #endif 33 | } FormData_pipeline_stream; 34 | 35 | typedef FormData_pipeline_stream *Form_pipeline_stream; 36 | 37 | #define Natts_pipeline_stream 2 38 | #define Anum_pipeline_stream_relid 1 39 | #define Anum_pipeline_stream_queries 2 40 | 41 | #define OSREL_OLD_ROW "old" 42 | #define OSREL_NEW_ROW "new" 43 | #define OSREL_DELTA_ROW "delta" 44 | 45 | extern void SyncPipelineStreamReaders(void); 46 | 47 | extern void SyncPipelineStream(void); 48 | 49 | extern Bitmapset *GetAllStreamReaders(Oid relid); 50 | extern Bitmapset *GetLocalStreamReaders(Oid relid); 51 | 52 | extern bool RangeVarIsStream(RangeVar *rv, bool missing_ok); 53 | 54 | extern bool RelidIsStream(Oid relid); 55 | extern bool RelidIsOutputStream(Oid relid); 56 | 57 | 58 | extern Relation OpenPipelineStream(LOCKMODE mode); 59 | extern void ClosePipelineStream(Relation ps, LOCKMODE mode); 60 | extern void transformCreateStreamStmt(CreateForeignTableStmt *stmt); 61 | extern void CreatePipelineStreamEntry(CreateForeignTableStmt *stmt, Oid relid); 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/test/regress/sql/sw_expiration.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE sw_vacuum_stream (key text) SERVER pipelinedb; 2 | CREATE VIEW sw_vacuum AS SELECT key, COUNT(*) FROM sw_vacuum_stream WHERE arrival_timestamp > clock_timestamp() - interval '3 second' GROUP BY key; 3 | 4 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 5 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 6 | 7 | SELECT pg_sleep(1); 8 | 9 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 10 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 11 | 12 | SELECT * FROM sw_vacuum ORDER BY key; 13 | 14 | -- Just verify that the mrel has more rows, as we can't gaurantee which time bucket the 15 | -- rows will fall in which makes it tricky to compare this result to a predetermined result 16 | SELECT (SELECT COUNT(*) FROM sw_vacuum) < (SELECT COUNT(*) FROM sw_vacuum_mrel); 17 | SELECT DISTINCT key FROM sw_vacuum_mrel ORDER BY key; 18 | 19 | SELECT pg_sleep(3); 20 | SELECT 0 * pipelinedb.ttl_expire('sw_vacuum'); 21 | 22 | SELECT * FROM sw_vacuum ORDER BY key; 23 | SELECT key, SUM(count) FROM sw_vacuum_mrel GROUP BY key ORDER BY key; 24 | 25 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 26 | 27 | SELECT pg_sleep(1); 28 | 29 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 30 | 31 | SELECT * FROM sw_vacuum ORDER BY key; 32 | SELECT (SELECT COUNT(*) FROM sw_vacuum) < (SELECT COUNT(*) FROM sw_vacuum_mrel); 33 | SELECT DISTINCT key FROM sw_vacuum_mrel ORDER BY key; 34 | 35 | SELECT 0 * pipelinedb.ttl_expire('sw_vacuum'); 36 | SELECT * FROM sw_vacuum ORDER BY key; 37 | SELECT key, SUM(count) FROM sw_vacuum_mrel GROUP BY key ORDER BY key; 38 | 39 | SELECT pg_sleep(3); 40 | SELECT 0 * pipelinedb.ttl_expire('sw_vacuum'); 41 | 42 | SELECT * FROM sw_vacuum ORDER BY key; 43 | SELECT key, SUM(count) FROM sw_vacuum_mrel GROUP BY key ORDER BY key; 44 | 45 | DROP FOREIGN TABLE sw_vacuum_stream CASCADE; 46 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_dist_agg.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_dist_agg_stream (x integer) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_dist_agg0 AS 4 | SELECT x % 10 AS g, dist_agg(x) 5 | FROM cont_dist_agg_stream 6 | GROUP BY g; 7 | 8 | CREATE VIEW test_dist_agg1 AS 9 | SELECT x % 10 AS g, dist_agg(x) 10 | FROM cont_dist_agg_stream 11 | GROUP BY g; 12 | 13 | INSERT INTO cont_dist_agg_stream (x) SELECT x FROM generate_series(1, 1000) x; 14 | 15 | SELECT g, dist_cdf(dist_agg, 50) FROM test_dist_agg0 ORDER BY g; 16 | SELECT g, dist_cdf(dist_agg, 500) FROM test_dist_agg0 ORDER BY g; 17 | SELECT g, dist_cdf(dist_agg, 10) FROM test_dist_agg1 ORDER BY g; 18 | SELECT g, dist_cdf(dist_agg, 900) FROM test_dist_agg1 ORDER BY g; 19 | 20 | SELECT g, dist_quantile(dist_agg, 0.1) FROM test_dist_agg0 ORDER BY g; 21 | SELECT g, dist_quantile(dist_agg, 0.25) FROM test_dist_agg0 ORDER BY g; 22 | SELECT g, dist_quantile(dist_agg, 0.50) FROM test_dist_agg1 ORDER BY g; 23 | SELECT g, dist_quantile(dist_agg, 0.99) FROM test_dist_agg1 ORDER BY g; 24 | 25 | INSERT INTO cont_dist_agg_stream (x) SELECT x FROM generate_series(1000, 5000) x; 26 | 27 | SELECT g, dist_cdf(dist_agg, 50) FROM test_dist_agg0 ORDER BY g; 28 | SELECT g, dist_cdf(dist_agg, 500) FROM test_dist_agg0 ORDER BY g; 29 | SELECT g, dist_cdf(dist_agg, 10) FROM test_dist_agg1 ORDER BY g; 30 | SELECT g, dist_cdf(dist_agg, 900) FROM test_dist_agg1 ORDER BY g; 31 | 32 | SELECT g, dist_quantile(dist_agg, 0.1) FROM test_dist_agg0 ORDER BY g; 33 | SELECT g, dist_quantile(dist_agg, 0.25) FROM test_dist_agg0 ORDER BY g; 34 | SELECT g, dist_quantile(dist_agg, 0.50) FROM test_dist_agg1 ORDER BY g; 35 | SELECT g, dist_quantile(dist_agg, 0.99) FROM test_dist_agg1 ORDER BY g; 36 | 37 | CREATE VIEW empty_regress WITH (sw = '1 day') AS 38 | SELECT dist_agg(x) FILTER (WHERE x > 30) FROM cont_dist_agg_stream; 39 | 40 | SELECT * FROM empty_regress; 41 | 42 | DROP FOREIGN TABLE cont_dist_agg_stream CASCADE; 43 | -------------------------------------------------------------------------------- /include/catalog.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * catalog.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef PIPELINE_CATALOG_H 11 | #define PIPELINE_CATALOG_H 12 | 13 | #include "access/attnum.h" 14 | #include "access/htup.h" 15 | #include "utils/relcache.h" 16 | 17 | enum PipelineSysCacheIdentifier 18 | { 19 | PIPELINEQUERYID, 20 | PIPELINEQUERYRELID, 21 | PIPELINEQUERYDEFRELID, 22 | PIPELINEQUERYMATRELID, 23 | PIPELINEQUERYOSRELID, 24 | PIPELINEQUERYSEQRELID, 25 | PIPELINEQUERYPKIDXID, 26 | PIPELINEQUERYLOOKUPIDXID, 27 | PIPELINESTREAMRELID, 28 | PIPELINESTREAMOID, 29 | PGAGGCOMBINEFN, 30 | PGAGGPARTIALCOMBINEFN, 31 | PIPELINECOMBINEAGGFN 32 | }; 33 | 34 | extern void InitPipelineCatalog(void); 35 | extern void PipelineCatalogInvalidate(int id); 36 | extern void PipelineCatalogInvalidateAll(void); 37 | 38 | extern void PipelineCatalogTupleInsert(Relation rel, HeapTuple tup); 39 | extern void PipelineCatalogTupleUpdate(Relation rel, ItemPointer otid, HeapTuple tup); 40 | extern void PipelineCatalogTupleDelete(Relation rel, ItemPointer tid); 41 | extern HeapTuple PipelineCatalogLookup(int id, int nkeys, ...); 42 | extern HeapTuple PipelineCatalogLookupForUpdate(Relation rel, int id, Datum key); 43 | 44 | extern Datum PipelineCatalogGetAttr(int cacheId, HeapTuple tup, AttrNumber attr, bool *isnull); 45 | 46 | extern Oid GetHashGroupOid(void); 47 | extern Oid GetLSHashGroupOid(void); 48 | extern Oid GetInsertIntoStreamOid(void); 49 | 50 | extern Oid GetPipelineQueryOid(void); 51 | extern Oid GetPipelineCombineOid(void); 52 | extern Oid GetPipelineStreamOid(void); 53 | extern Oid GetPipelineExecLockOid(void); 54 | extern Oid GetDeserializeOid(void); 55 | extern bool CreateExtensionIsForPipelineDB(Node *node); 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /src/test/regress/parallel_schedule: -------------------------------------------------------------------------------- 1 | # 2 | # Parallel group of tests 3 | # 4 | test: pipeline_stream 5 | 6 | # 7 | # Parallel group of tests 8 | # 9 | test: cont_limit cont_transform cont_view_sanity cont_sum cont_avg cont_count 10 | 11 | # 12 | # Parallel group of tests 13 | # 14 | test: stream_targets date_round stream_insert_subselect stream_exprs prepared_stream_insert cont_view_namespace 15 | 16 | # 17 | # Parallel group of tests 18 | # 19 | test: cont_bool_agg cont_min_max cont_regr cont_pk matrel_constraints cont_json_agg 20 | 21 | # 22 | # Parallel group of tests 23 | # 24 | # This test can be sensitive to expected failures in other tests, so we give it its own group 25 | test: user_combine 26 | 27 | # 28 | # Parallel group of tests 29 | # 30 | test: cont_sw_avg cont_stats_agg cont_array_agg cont_set_agg cont_distinct cont_hll_agg hll 31 | 32 | # 33 | # Parallel group of tests 34 | # 35 | test: cont_hs_agg cont_os_agg typed_streams bloom cont_complex_types topk cont_topk_agg 36 | 37 | # 38 | # Parallel group of tests 39 | # 40 | test: dist cont_dist_agg freq cont_freq_agg keyed_min_max first_values bucket_agg output_streams 41 | 42 | # 43 | # Parallel group of tests 44 | # 45 | test: delta_streams stream_table_join cont_activate cont_subselect hash_group 46 | 47 | # 48 | # Parallel group of tests 49 | # 50 | test: ttl_expiration sw_expiration part_ttl 51 | 52 | # 53 | # Parallel group of tests 54 | # 55 | test: analyze_cont_view 56 | 57 | # 58 | # Parallel group of tests 59 | # 60 | test: cont_stats cont_sw_sum cont_sw_bool_agg cont_sw_min_max cont_sw_regr cont_sw_stats 61 | 62 | # 63 | # Parallel group of tests 64 | # 65 | test: cont_sw_count cont_object_agg cont_sw_hs_agg cont_sw_os_agg create_cont_view 66 | 67 | # 68 | # Parallel group of tests 69 | # 70 | test: cont_alter cont_sw_object_agg pipeline_regress cont_grouping_sets cont_matrel cont_index 71 | 72 | # 73 | # Parallel group of tests 74 | # 75 | test: cont_part_basic cont_part_regress 76 | -------------------------------------------------------------------------------- /include/matrel.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * matrel.h 4 | * Interface for modifying continuous view materialization tables 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef CQMATVIEW_H 12 | #define CQMATVIEW_H 13 | 14 | #include "postgres.h" 15 | 16 | #include "nodes/execnodes.h" 17 | #include "postgres_ext.h" 18 | 19 | typedef struct MatRelPartitionKey 20 | { 21 | Datum value; 22 | Oid typid; 23 | } MatRelPartitionKey; 24 | 25 | extern bool matrels_writable; 26 | extern bool log_partitions; 27 | 28 | #define CQ_OSREL_SUFFIX "_osrel" 29 | #define CQ_MATREL_SUFFIX "_mrel" 30 | #define CQ_SEQREL_SUFFIX "_seq" 31 | #define CQ_DEFREL_SUFFIX "_def" 32 | #define CQ_MATREL_PKEY "$pk" 33 | #define MatRelWritable() (matrels_writable) 34 | 35 | extern ResultRelInfo *CQMatRelOpen(Relation matrel); 36 | extern void CQOSRelClose(ResultRelInfo *rinfo); 37 | extern ResultRelInfo *CQOSRelOpen(Relation osrel); 38 | extern void CQMatRelClose(ResultRelInfo *rinfo); 39 | extern void ExecInsertCQMatRelIndexTuples(ResultRelInfo *indstate, 40 | TupleTableSlot *slot, ItemPointer tid, 41 | EState *estate); 42 | extern pg_nodiscard bool ExecCQMatRelUpdate(ResultRelInfo *ri, 43 | TupleTableSlot *slot, 44 | EState *estate); 45 | extern pg_nodiscard bool ExecCQMatRelInsert(ResultRelInfo *ri, 46 | TupleTableSlot *slot, 47 | EState *estate); 48 | extern void DefineMatRelPartition(RangeVar *matrel, 49 | MatRelPartitionKey *lower_bound, 50 | Interval *duration); 51 | 52 | extern char *CVNameToOSRelName(char *cv_name); 53 | extern char *CVNameToMatRelName(char *cv_name); 54 | extern char *CVNameToDefRelName(char *cv_name); 55 | extern char *CVNameToSeqRelName(char *cv_name); 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /src/test/regress/sql/stream_insert_subselect.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE stream_subselect_t (x integer); 2 | INSERT INTO stream_subselect_t (SELECT generate_series(1, 100)); 3 | 4 | CREATE FOREIGN TABLE stream_subselect_stream (x int, price int, t timestamptz) SERVER pipelinedb; 5 | CREATE VIEW stream_subselect_v0 AS SELECT x::integer FROM stream_subselect_stream; 6 | 7 | INSERT INTO stream_subselect_stream (x) (SELECT * FROM stream_subselect_t); 8 | INSERT INTO stream_subselect_stream (x) (SELECT * FROM (SELECT x AS y FROM stream_subselect_t) s0); 9 | 10 | SELECT COUNT(DISTINCT x) FROM stream_subselect_v0; 11 | 12 | CREATE VIEW stream_subselect_v1 AS SELECT x::integer, COUNT(*) FROM stream_subselect_stream GROUP BY x; 13 | INSERT INTO stream_subselect_stream (x) (SELECT generate_series(1, 1000)); 14 | 15 | SELECT COUNT(DISTINCT x) FROM stream_subselect_v1; 16 | 17 | -- It's not possible to SELECT from another stream in a stream INSERT 18 | CREATE FOREIGN TABLE stream_subselect_s0 (x integer) SERVER pipelinedb; 19 | INSERT INTO stream_subselect_stream (x) (SELECT x FROM stream_subselect_s0); 20 | 21 | CREATE VIEW stream_subselect_v2 AS SELECT x::integer FROM stream_subselect_stream; 22 | INSERT INTO stream_subselect_stream (x) (SELECT generate_series(1, 1000) AS x ORDER BY random()); 23 | 24 | SELECT COUNT(DISTINCT x) FROM stream_subselect_v2; 25 | 26 | CREATE VIEW stream_subselect_v3 AS SELECT x::integer FROM stream_subselect_stream; 27 | INSERT INTO stream_subselect_stream (x) (SELECT * FROM stream_subselect_t WHERE x IN (SELECT generate_series(1, 20))); 28 | 29 | SELECT * FROM stream_subselect_v3 ORDER BY x; 30 | 31 | CREATE VIEW stream_subselect_v4 AS SELECT COUNT(*) FROM stream_subselect_stream; 32 | INSERT INTO stream_subselect_stream (price, t) SELECT 10 + random() AS price, current_timestamp - interval '1 minute' * random() AS t FROM generate_series(1, 1000); 33 | 34 | SELECT * FROM stream_subselect_v4; 35 | 36 | DROP FOREIGN TABLE stream_subselect_stream CASCADE; 37 | DROP TABLE stream_subselect_t; 38 | -------------------------------------------------------------------------------- /src/test/regress/expected/part_ttl.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE ttl_stream (x integer) SERVER pipelinedb; 2 | CREATE VIEW ttl0 WITH (ttl='3 seconds', ttl_column='ts', 3 | partition_by='ts', partition_duration='1 second') 4 | AS SELECT arrival_timestamp AS ts, x FROM ttl_stream; 5 | SELECT c.relname, ttl, ttl_attno FROM pipelinedb.cont_query cq 6 | JOIN pg_class c ON c.oid = cq.relid WHERE c.relname = 'ttl0' 7 | ORDER BY c.relname; 8 | relname | ttl | ttl_attno 9 | ---------+-----+----------- 10 | ttl0 | 3 | 1 11 | (1 row) 12 | 13 | INSERT INTO ttl_stream (x) VALUES (0); 14 | INSERT INTO ttl_stream (x) VALUES (1); 15 | INSERT INTO ttl_stream (x) VALUES (2); 16 | SELECT x, "$pk" FROM ttl0_mrel ORDER BY ts; 17 | x | $pk 18 | ---+----- 19 | 0 | 1 20 | 1 | 2 21 | 2 | 3 22 | (3 rows) 23 | 24 | SELECT pg_sleep(3); 25 | pg_sleep 26 | ---------- 27 | 28 | (1 row) 29 | 30 | SELECT 0 * pipelinedb.ttl_expire('ttl0'); 31 | ?column? 32 | ---------- 33 | 0 34 | (1 row) 35 | 36 | SELECT x, "$pk" FROM ttl0_mrel ORDER BY ts; 37 | x | $pk 38 | ---+----- 39 | 0 | 1 40 | 1 | 2 41 | 2 | 3 42 | (3 rows) 43 | 44 | INSERT INTO ttl_stream (x) VALUES (0); 45 | INSERT INTO ttl_stream (x) VALUES (1); 46 | INSERT INTO ttl_stream (x) VALUES (2); 47 | SELECT x, "$pk" FROM ttl0_mrel ORDER BY ts; 48 | x | $pk 49 | ---+----- 50 | 0 | 1 51 | 1 | 2 52 | 2 | 3 53 | 0 | 4 54 | 1 | 5 55 | 2 | 6 56 | (6 rows) 57 | 58 | SELECT pg_sleep(3); 59 | pg_sleep 60 | ---------- 61 | 62 | (1 row) 63 | 64 | SELECT 0 * pipelinedb.ttl_expire('ttl0'); 65 | ?column? 66 | ---------- 67 | 0 68 | (1 row) 69 | 70 | SELECT x, "$pk" FROM ttl0_mrel ORDER BY ts; 71 | x | $pk 72 | ---+----- 73 | 0 | 1 74 | 1 | 2 75 | 2 | 3 76 | 0 | 4 77 | 1 | 5 78 | 2 | 6 79 | (6 rows) 80 | 81 | DROP VIEW ttl0; 82 | CREATE VIEW ttl2 WITH (partition_by='ts', partition_duration='1 second') 83 | AS SELECT arrival_timestamp AS ts, x FROM ttl_stream; 84 | DROP VIEW ttl2; 85 | DROP FOREIGN TABLE ttl_stream CASCADE; 86 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_activate.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_activate_s (x int) SERVER pipelinedb; 2 | CREATE VIEW cont_activate_v0 AS SELECT count(*) FROM cont_activate_s; 3 | CREATE VIEW cont_activate_v1 AS SELECT count(*) FROM cont_activate_s; 4 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 5 | SELECT * FROM cont_activate_v0; 6 | count 7 | ------- 8 | 100 9 | (1 row) 10 | 11 | SELECT * FROM cont_activate_v1; 12 | count 13 | ------- 14 | 100 15 | (1 row) 16 | 17 | SELECT pipelinedb.deactivate('cont_activate_v0'); 18 | deactivate 19 | ------------ 20 | t 21 | (1 row) 22 | 23 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 24 | SELECT * FROM cont_activate_v0; 25 | count 26 | ------- 27 | 100 28 | (1 row) 29 | 30 | SELECT * FROM cont_activate_v1; 31 | count 32 | ------- 33 | 200 34 | (1 row) 35 | 36 | SELECT pipelinedb.deactivate('cont_activate_v1'); 37 | deactivate 38 | ------------ 39 | t 40 | (1 row) 41 | 42 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 43 | SELECT * FROM cont_activate_v0; 44 | count 45 | ------- 46 | 100 47 | (1 row) 48 | 49 | SELECT * FROM cont_activate_v1; 50 | count 51 | ------- 52 | 200 53 | (1 row) 54 | 55 | SELECT pipelinedb.activate('cont_activate_v0'); 56 | activate 57 | ---------- 58 | t 59 | (1 row) 60 | 61 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 62 | SELECT * FROM cont_activate_v0; 63 | count 64 | ------- 65 | 200 66 | (1 row) 67 | 68 | SELECT * FROM cont_activate_v1; 69 | count 70 | ------- 71 | 200 72 | (1 row) 73 | 74 | SELECT pipelinedb.activate('cont_activate_v1'); 75 | activate 76 | ---------- 77 | t 78 | (1 row) 79 | 80 | INSERT INTO cont_activate_s (x) SELECT generate_series(1, 100) AS x; 81 | SELECT * FROM cont_activate_v0; 82 | count 83 | ------- 84 | 300 85 | (1 row) 86 | 87 | SELECT * FROM cont_activate_v1; 88 | count 89 | ------- 90 | 300 91 | (1 row) 92 | 93 | DROP VIEW cont_activate_v0; 94 | DROP VIEW cont_activate_v1; 95 | DROP FOREIGN TABLE cont_activate_s CASCADE; 96 | -------------------------------------------------------------------------------- /src/test/regress/expected/prepared_stream_insert.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE prep_insert_stream (x float8, y int, z int) SERVER pipelinedb; 2 | CREATE VIEW prep_insert0 AS SELECT COUNT(*) FROM prep_insert_stream; 3 | CREATE VIEW prep_insert1 AS SELECT sum(x::float8) AS fsum, sum(y::int8) AS isum FROM prep_insert_stream; 4 | CREATE VIEW prep_insert2 AS SELECT sum(x::integer) AS isum, sum(y::int4) AS i4sum FROM prep_insert_stream; 5 | CREATE TABLE prep_insert_t (x integer, y integer, z integer); 6 | PREPARE prep0 AS INSERT INTO prep_insert_stream (x) VALUES ($1); 7 | PREPARE prep1 AS INSERT INTO prep_insert_stream (x, y) VALUES ($1, $2); 8 | PREPARE prep_t AS INSERT INTO prep_insert_t (x, y, z) VALUES ($1, $2, $3); 9 | EXECUTE prep0(1); 10 | EXECUTE prep0(1); 11 | EXECUTE prep0(1); 12 | EXECUTE prep0(1); 13 | EXECUTE prep0(1); 14 | EXECUTE prep0(1); 15 | EXECUTE prep0(1); 16 | EXECUTE prep0(1.5); 17 | EXECUTE prep0(1.5); 18 | EXECUTE prep0(1.5); 19 | EXECUTE prep0(1.5); 20 | EXECUTE prep0(1.5); 21 | EXECUTE prep0(1.5); 22 | EXECUTE prep0(1.5); 23 | EXECUTE prep1(1, 1.1); 24 | EXECUTE prep1(1, 1.1); 25 | EXECUTE prep1(1, 1.1); 26 | EXECUTE prep1(1, 1.1); 27 | EXECUTE prep1(1, 1.1); 28 | EXECUTE prep1(1, 1.1); 29 | EXECUTE prep1(1, 1.1); 30 | EXECUTE prep1(1, 1.1); 31 | EXECUTE prep1(1, 1.1); 32 | EXECUTE prep1(1, 1.1); 33 | EXECUTE prep_t(0, 0, 0); 34 | EXECUTE prep_t(0, 0, 0); 35 | EXECUTE prep_t(0, 0, 0); 36 | EXECUTE prep_t(0, 0, 0); 37 | EXECUTE prep_t(0, 0, 0); 38 | EXECUTE prep_t(0, 0, 0); 39 | EXECUTE prep_t(0, 0, 0); 40 | EXECUTE prep_t(0, 0, 0); 41 | SELECT * FROM prep_insert0; 42 | count 43 | ------- 44 | 24 45 | (1 row) 46 | 47 | SELECT * FROM prep_insert1; 48 | fsum | isum 49 | ------+------ 50 | 27.5 | 10 51 | (1 row) 52 | 53 | SELECT * FROM prep_insert2; 54 | isum | i4sum 55 | ------+------- 56 | 31 | 10 57 | (1 row) 58 | 59 | DROP FOREIGN TABLE prep_insert_stream CASCADE; 60 | NOTICE: drop cascades to 3 other objects 61 | DETAIL: drop cascades to view prep_insert0 62 | drop cascades to view prep_insert1 63 | drop cascades to view prep_insert2 64 | DROP TABLE prep_insert_t; 65 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_array_agg.sql: -------------------------------------------------------------------------------- 1 | -- array_sort function 2 | CREATE FUNCTION 3 | ca_array_sort( 4 | array_vals_to_sort anyarray 5 | ) 6 | RETURNS TABLE ( 7 | sorted_array anyarray 8 | ) 9 | AS $BODY$ 10 | BEGIN 11 | RETURN QUERY SELECT 12 | ARRAY_AGG(val) AS sorted_array 13 | FROM 14 | ( 15 | SELECT 16 | UNNEST(array_vals_to_sort) AS val 17 | ORDER BY 18 | val 19 | ) AS sorted_vals 20 | ; 21 | END; 22 | $BODY$ 23 | LANGUAGE plpgsql; 24 | 25 | -- array_agg 26 | CREATE FOREIGN TABLE cqobjectagg_stream (k text, v int) SERVER pipelinedb; 27 | 28 | CREATE VIEW test_array_agg AS SELECT k::text, array_agg(v::integer) FROM cqobjectagg_stream GROUP BY k; 29 | \d+ test_array_agg_mrel 30 | 31 | INSERT INTO cqobjectagg_stream (k, v) VALUES ('x', 0), ('x', 1), ('x', 2), ('x', 3); 32 | INSERT INTO cqobjectagg_stream (k, v) VALUES ('y', 0), ('y', 1); 33 | 34 | SELECT k, ca_array_sort(array_agg) FROM test_array_agg ORDER BY k; 35 | 36 | INSERT INTO cqobjectagg_stream (k, v) VALUES ('x', 4), ('y', 2), ('z', 10), ('z', 20); 37 | 38 | SELECT k, ca_array_sort(array_agg) FROM test_array_agg ORDER BY k; 39 | 40 | DROP VIEW test_array_agg; 41 | 42 | CREATE VIEW test_array_agg AS SELECT array_agg(k::text) FROM cqobjectagg_stream; 43 | 44 | INSERT INTO cqobjectagg_stream (k) VALUES ('hello'), ('world'); 45 | SELECT pg_sleep(0.1); 46 | INSERT INTO cqobjectagg_stream (k) VALUES ('lol'), ('cat'); 47 | 48 | SELECT ca_array_sort(array_agg) FROM test_array_agg; 49 | DROP FOREIGN TABLE cqobjectagg_stream CASCADE; 50 | 51 | -- array_agg_array 52 | CREATE FOREIGN TABLE cqobjectagg_stream (x int, y int) SERVER pipelinedb; 53 | 54 | CREATE VIEW test_array_agg_array AS SELECT array_agg(ARRAY[x::int, y::int]) FROM cqobjectagg_stream; 55 | 56 | INSERT INTO cqobjectagg_stream (x, y) VALUES (1, 11); 57 | INSERT INTO cqobjectagg_stream (x, y) VALUES (2, 12); 58 | SELECT pg_sleep(0.1); 59 | INSERT INTO cqobjectagg_stream (x, y) VALUES (3, 13); 60 | 61 | SELECT * FROM test_array_agg_array; 62 | 63 | DROP FOREIGN TABLE cqobjectagg_stream CASCADE; -------------------------------------------------------------------------------- /src/test/regress/expected/dist.out: -------------------------------------------------------------------------------- 1 | SELECT dist_quantile(dist_agg(x), 0.50) FROM generate_series(1, 1000) AS x; 2 | dist_quantile 3 | --------------- 4 | 500.5 5 | (1 row) 6 | 7 | SELECT dist_quantile(dist_agg(x), 0.10) FROM generate_series(1, 1000) AS x; 8 | dist_quantile 9 | --------------- 10 | 100.5 11 | (1 row) 12 | 13 | SELECT dist_quantile(dist_agg(x), 0.05) FROM generate_series(1, 1000) AS x; 14 | dist_quantile 15 | --------------- 16 | 50.5 17 | (1 row) 18 | 19 | SELECT dist_quantile(dist_agg(x), 0.90) FROM generate_series(1, 1000) AS x; 20 | dist_quantile 21 | --------------- 22 | 900.5 23 | (1 row) 24 | 25 | SELECT dist_quantile(dist_agg(x), 0.99) FROM generate_series(1, 1000) AS x; 26 | dist_quantile 27 | --------------- 28 | 990.5 29 | (1 row) 30 | 31 | SELECT dist_cdf(dist_agg(x), 1) FROM generate_series(1, 1000) AS x; 32 | dist_cdf 33 | ---------- 34 | 0 35 | (1 row) 36 | 37 | SELECT dist_cdf(dist_agg(x), 5) FROM generate_series(1, 1000) AS x; 38 | dist_cdf 39 | ---------- 40 | 0.0045 41 | (1 row) 42 | 43 | SELECT dist_cdf(dist_agg(x), 10) FROM generate_series(1, 1000) AS x; 44 | dist_cdf 45 | ---------- 46 | 0.0095 47 | (1 row) 48 | 49 | SELECT dist_cdf(dist_agg(x), 500) FROM generate_series(1, 1000) AS x; 50 | dist_cdf 51 | ---------- 52 | 0.4995 53 | (1 row) 54 | 55 | SELECT dist_cdf(dist_agg(x), 999) FROM generate_series(1, 1000) AS x; 56 | dist_cdf 57 | ---------- 58 | 0.9985 59 | (1 row) 60 | 61 | SELECT dist_cdf(dist_agg(x), 1000) FROM generate_series(1, 1000) AS x; 62 | dist_cdf 63 | ---------- 64 | 1 65 | (1 row) 66 | 67 | SELECT dist_cdf(dist_agg(x), 2000) FROM generate_series(1, 1000) AS x; 68 | dist_cdf 69 | ---------- 70 | 1 71 | (1 row) 72 | 73 | SELECT dist_quantile(dist_add(dist_agg(x), 10000), 0.99) FROM generate_series(1, 1000) AS x; 74 | dist_quantile 75 | --------------- 76 | 991.49 77 | (1 row) 78 | 79 | SELECT dist_cdf(dist_add(dist_agg(x), 10000), 2000) FROM generate_series(1, 1000) AS x; 80 | dist_cdf 81 | -------------------- 82 | 0.9982240845478263 83 | (1 row) 84 | 85 | -------------------------------------------------------------------------------- /src/test/regress/sql/hll.sql: -------------------------------------------------------------------------------- 1 | SELECT hll_cardinality(hll_agg('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' || x)) FROM generate_series(1, 100) AS x; 2 | SELECT hll_cardinality(hll_agg(x * 1.1)) FROM generate_series(1, 100) AS x; 3 | SELECT hll_cardinality(hll_agg(x::numeric)) FROM generate_series(1, 100) AS x; 4 | SELECT hll_cardinality(hll_agg(x)) FROM generate_series(1, 10000) AS x; 5 | SELECT hll_cardinality(hll_agg(x)) FROM generate_series(1, 100000) AS x; 6 | 7 | SELECT hll_cardinality(hll_union_agg(hll_agg)) FROM (SELECT hll_agg(x) FROM generate_series(1, 100) AS x UNION ALL SELECT hll_agg(x) FROM generate_series(101, 200) AS x) _; 8 | SELECT hll_cardinality(hll_union_agg(hll_agg)) FROM (SELECT hll_agg(x) FROM generate_series(1, 100) AS x UNION ALL SELECT hll_agg(x) FROM generate_series(101, 2000) AS x) _; 9 | 10 | SELECT hll_cardinality( 11 | hll_union( 12 | (SELECT hll_agg(x) FROM generate_series(1, 1000) AS x), 13 | (SELECT hll_agg(x) FROM generate_series(100, 1100) AS x), 14 | NULL)); 15 | 16 | -- Different HLL dimensions 17 | SELECT hll_cardinality( 18 | hll_union( 19 | (SELECT hll_agg(x, 12) FROM generate_series(1, 1000) AS x), 20 | (SELECT hll_agg(x) FROM generate_series(100, 1100) AS x))); 21 | 22 | -- Wrong type 23 | SELECT hll_cardinality( 24 | hll_union( 25 | 'not an hll', 26 | (SELECT hll_agg(x) FROM generate_series(100, 1100) AS x))); 27 | 28 | -- Byref but not varlena types 29 | CREATE TABLE byref (uuid uuid, name name); 30 | INSERT INTO byref (uuid, name) VALUES ('fe636e28-db82-43af-ac48-df26a4cda1f3', 'alice'); 31 | INSERT INTO byref (uuid, name) VALUES ('fe636e28-db82-43af-ac48-df26a4cda1f3', 'alice'); 32 | 33 | SELECT hll_cardinality(hll_agg(uuid)) FROM byref; 34 | SELECT hll_cardinality(hll_agg(name)) FROM byref; 35 | 36 | INSERT INTO byref (uuid, name) VALUES ('fff36e28-db82-43af-ac48-df26a4cda1f3', 'bob'); 37 | INSERT INTO byref (uuid, name) VALUES ('fff36e28-db82-43af-ac48-df26a4cda1f3', 'bob'); 38 | 39 | SELECT hll_cardinality(hll_agg(uuid)) FROM byref; 40 | SELECT hll_cardinality(hll_agg(name)) FROM byref; 41 | -------------------------------------------------------------------------------- /src/test/py/test_sync.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import getpass 3 | import psycopg2 4 | import threading 5 | import time 6 | 7 | 8 | def test_userset_sync(pipeline, clean_db): 9 | 10 | pipeline.create_stream('s', x='int') 11 | pipeline.create_cv('sync', 'SELECT count(*) FROM s WHERE x = 0') 12 | pipeline.create_cv('async', 'SELECT count(*) FROM s WHERE x = 1') 13 | pipeline.create_cv('delay', 'SELECT x::int, cq_sleep(0.1) FROM s') 14 | 15 | NUM_INSERTS = 100 16 | 17 | def insert(sync): 18 | conn = psycopg2.connect('dbname=postgres user=%s host=localhost port=%s' 19 | % (getpass.getuser(), pipeline.port)) 20 | cur = conn.cursor() 21 | cur.execute('SET pipelinedb.stream_insert_level=sync_%s' % 22 | ('commit' if sync else 'receive')) 23 | for i in range(NUM_INSERTS): 24 | try: 25 | cur.execute('INSERT INTO s (x) VALUES (%d)' % (0 if sync else 1)) 26 | conn.commit() 27 | except psycopg2.errors.UndefinedTable: 28 | pass 29 | conn.close() 30 | 31 | sync = threading.Thread(target=insert, args=(True,)) 32 | _async = threading.Thread(target=insert, args=(False,)) 33 | 34 | start = time.time() 35 | 36 | sync.start() 37 | _async.start() 38 | 39 | _async.join() 40 | async_time = time.time() - start 41 | assert async_time < NUM_INSERTS * 0.1 42 | 43 | num_sync = pipeline.execute('SELECT count FROM sync')[0]['count'] 44 | num_async = pipeline.execute('SELECT count FROM async')[0]['count'] 45 | total = pipeline.execute('SELECT count(*) FROM delay')[0]['count'] 46 | assert num_async < NUM_INSERTS 47 | assert num_sync < NUM_INSERTS 48 | assert total < NUM_INSERTS * 2 49 | 50 | sync.join() 51 | assert time.time() - start > (NUM_INSERTS * 0.08 + async_time) 52 | 53 | pipeline.execute('COMMIT') 54 | num_sync = pipeline.execute('SELECT count FROM sync')[0]['count'] 55 | num_async = pipeline.execute('SELECT count FROM async')[0]['count'] 56 | total = pipeline.execute('SELECT count(*) FROM delay')[0]['count'] 57 | assert num_sync == NUM_INSERTS 58 | assert num_async == NUM_INSERTS 59 | assert total == NUM_INSERTS * 2 60 | -------------------------------------------------------------------------------- /include/miscutils.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * Miscellaneous utilities 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | #ifndef MISCUTILS_H 11 | #define MISCUTILS_H 12 | 13 | #include "postgres.h" 14 | 15 | #include "access/htup_details.h" 16 | #include "catalog/pg_type.h" 17 | #include "executor/tuptable.h" 18 | #include "miscadmin.h" 19 | #include "lib/stringinfo.h" 20 | #include "signal.h" 21 | #include "storage/dsm.h" 22 | #include "utils/typcache.h" 23 | 24 | extern volatile sig_atomic_t pipeline_got_SIGTERM; 25 | 26 | typedef struct tagged_ref_t 27 | { 28 | void *ptr; 29 | uint64 tag; 30 | } tagged_ref_t; 31 | 32 | #define set_sigterm_flag() \ 33 | do \ 34 | { \ 35 | pipeline_got_SIGTERM = true; \ 36 | } while (0) 37 | #define get_sigterm_flag() (pipeline_got_SIGTERM) 38 | 39 | #define ptr_difference(begin, end) ((void *) (((char *) end) - ((char *) begin))) 40 | #define ptr_offset(begin, offset) ((void *) (((char *) begin) + ((uintptr_t) offset))) 41 | 42 | #define foreach_tuple(slot, store) \ 43 | while (tuplestore_gettupleslot(store, true, false, slot)) 44 | 45 | #define BITMAPSET_SIZE(nwords) \ 46 | (offsetof(Bitmapset, words) + (nwords) * sizeof(bitmapword)) 47 | 48 | extern void append_suffix(char *str, char *suffix, int max_len); 49 | extern int skip_token(const char *str, char* substr, int start); 50 | extern char *random_hex(int len); 51 | 52 | /* hash functions */ 53 | extern void MurmurHash3_128(const void *key, const Size len, const uint64_t seed, void *out); 54 | extern uint64_t MurmurHash3_64(const void *key, const Size len, const uint64_t seed); 55 | extern void SlotAttrsToBytes(TupleTableSlot *slot, int num_attrs, AttrNumber *attrs, StringInfo buf); 56 | extern void DatumToBytes(Datum d, TypeCacheEntry *typ, StringInfo buf); 57 | 58 | extern Oid GetTypeOid(char *name); 59 | 60 | extern bool equalTupleDescsWeak(TupleDesc tupdesc1, TupleDesc tupdesc2, bool check_names); 61 | 62 | extern void print_tupledesc(TupleDesc desc); 63 | 64 | #endif /* MISCUTILS_H */ 65 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_matrel.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cont_matrel_stream (x int) SERVER pipelinedb; 2 | CREATE VIEW cont_matrel AS SELECT COUNT(*) FROM cont_matrel_stream; 3 | INSERT INTO cont_matrel_stream (x) VALUES (1); 4 | SELECT * FROM cont_matrel; 5 | count 6 | ------- 7 | 1 8 | (1 row) 9 | 10 | INSERT INTO cont_matrel_mrel (count) VALUES (1); 11 | ERROR: cannot change materialization table "cont_matrel_mrel" of continuous view "cont_matrel" 12 | HINT: Toggle the "pipelinedb.matrels_writable" parameter to change this behavior. 13 | UPDATE cont_matrel_mrel SET count = 2; 14 | ERROR: cannot change materialization table "cont_matrel_mrel" of continuous view "cont_matrel" 15 | HINT: Toggle the "pipelinedb.matrels_writable" parameter to change this behavior. 16 | DELETE FROM cont_matrel_mrel; 17 | ERROR: cannot change materialization table "cont_matrel_mrel" of continuous view "cont_matrel" 18 | HINT: Toggle the "pipelinedb.matrels_writable" parameter to change this behavior. 19 | SET pipelinedb.matrels_writable TO ON; 20 | UPDATE cont_matrel_mrel SET count = 2; 21 | SELECT * FROM cont_matrel; 22 | count 23 | ------- 24 | 2 25 | (1 row) 26 | 27 | INSERT INTO cont_matrel_stream (x) VALUES (1); 28 | SELECT * FROM cont_matrel; 29 | count 30 | ------- 31 | 3 32 | (1 row) 33 | 34 | DELETE FROM cont_matrel_mrel; 35 | SELECT * FROM cont_matrel; 36 | count 37 | ------- 38 | (0 rows) 39 | 40 | INSERT INTO cont_matrel_stream (x) VALUES (1); 41 | SELECT * FROM cont_matrel; 42 | count 43 | ------- 44 | 1 45 | (1 row) 46 | 47 | SELECT pipelinedb.truncate_continuous_view('cont_matrel'); 48 | truncate_continuous_view 49 | -------------------------- 50 | 51 | (1 row) 52 | 53 | INSERT INTO cont_matrel_mrel (count, "$pk") VALUES (5, 1); 54 | INSERT INTO cont_matrel_mrel (count, "$pk") VALUES (10, 1); 55 | ERROR: duplicate key value violates unique constraint "cont_matrel_mrel_pkey" 56 | DETAIL: Key ("$pk")=(1) already exists. 57 | SELECT * FROM cont_matrel; 58 | count 59 | ------- 60 | 5 61 | (1 row) 62 | 63 | INSERT INTO cont_matrel_stream (x) VALUES (1); 64 | SELECT * FROM cont_matrel; 65 | count 66 | ------- 67 | 6 68 | (1 row) 69 | 70 | DROP VIEW cont_matrel; 71 | DROP FOREIGN TABLE cont_matrel_stream CASCADE; 72 | -------------------------------------------------------------------------------- /src/test/py/test_dist_agg.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | 3 | 4 | def test_dist_agg(pipeline, clean_db): 5 | """ 6 | Test dist_agg, dist_merge_agg, dist_cdf, dist_quantile 7 | """ 8 | q = """ 9 | SELECT k::integer, dist_agg(x::int) AS t FROM test_tdigest_stream 10 | GROUP BY k 11 | """ 12 | desc = ('k', 'x') 13 | pipeline.create_stream('test_tdigest_stream', k='int', x='int') 14 | pipeline.create_cv('test_tdigest_agg', q) 15 | 16 | rows = [] 17 | for _ in range(10): 18 | for n in range(1000): 19 | rows.append((0, n)) 20 | rows.append((1, n + 500)) 21 | 22 | pipeline.insert('test_tdigest_stream', desc, rows) 23 | 24 | result = pipeline.execute('SELECT dist_quantile(t, 0.1) FROM test_tdigest_agg ORDER BY k') 25 | assert len(result) == 2 26 | assert abs(int(result[0]['dist_quantile']) - 99) <= 1 27 | assert abs(int(result[1]['dist_quantile']) - 599) <= 1 28 | 29 | result = pipeline.execute('SELECT dist_quantile(combine(t), 0.1) FROM test_tdigest_agg') 30 | assert len(result) == 1 31 | assert abs(int(result[0]['dist_quantile']) - 200) <= 4 32 | 33 | result = pipeline.execute('SELECT dist_cdf(t, 600) FROM test_tdigest_agg ORDER BY k') 34 | assert len(result) == 2 35 | assert round(result[0]['dist_cdf'], 2) == 0.6 36 | assert round(result[1]['dist_cdf'], 2) == 0.1 37 | 38 | result = pipeline.execute('SELECT dist_cdf(combine(t), 600) FROM test_tdigest_agg') 39 | assert len(result) == 1 40 | assert round(result[0]['dist_cdf'], 2) == 0.35 41 | 42 | def test_tdigest_type(pipeline, clean_db): 43 | pipeline.create_table('test_tdigest_type', x='int', y='tdigest') 44 | pipeline.execute('INSERT INTO test_tdigest_type (x, y) VALUES ' 45 | '(1, tdigest_empty()), (2, tdigest_empty())') 46 | 47 | for i in range(1000): 48 | pipeline.execute('UPDATE test_tdigest_type SET y = dist_add(y, %d %% (x * 500))' % i) 49 | 50 | result = list(pipeline.execute('SELECT dist_cdf(y, 400), ' 51 | 'dist_quantile(y, 0.9)' 52 | 'FROM test_tdigest_type ORDER BY x')) 53 | assert list(map(lambda x: round(x, 1), (result[0]['dist_cdf'], result[0]['dist_quantile']))) == [0.8, 449.5] 54 | assert list(map(lambda x: round(x, 1), (result[1]['dist_cdf'], result[1]['dist_quantile']))) == [0.4, 899.5] 55 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_bool_agg.sql: -------------------------------------------------------------------------------- 1 | -- bit_and, bit_or 2 | CREATE FOREIGN TABLE bit_stream_cqboolagg (k text, b bit) SERVER pipelinedb; 3 | 4 | CREATE VIEW test_bit_and AS SELECT k::text, bit_and(b::bit) FROM bit_stream_cqboolagg GROUP BY k; 5 | CREATE VIEW test_bit_or AS SELECT k::text, bit_or(b::bit) FROM bit_stream_cqboolagg GROUP BY k; 6 | 7 | INSERT INTO bit_stream_cqboolagg (k, b) VALUES ('x', 1::bit), ('x', '1'::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit); 8 | INSERT INTO bit_stream_cqboolagg (k, b) VALUES ('y', 0::bit), ('y', 1::bit), ('y', 0::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit); 9 | 10 | SELECT * FROM test_bit_and ORDER BY k DESC; 11 | SELECT * FROM test_bit_or ORDER BY k DESC; 12 | 13 | INSERT INTO bit_stream_cqboolagg (k, b) VALUES ('x', 1::bit), ('x', 0::bit); 14 | INSERT INTO bit_stream_cqboolagg (k, b) VALUES ('y', 0::bit); 15 | 16 | SELECT * FROM test_bit_and ORDER BY k DESC; 17 | SELECT * FROM test_bit_or ORDER BY k DESC; 18 | 19 | DROP FOREIGN TABLE bit_stream_cqboolagg CASCADE; 20 | 21 | -- bool_and, bool_or, every 22 | CREATE FOREIGN TABLE bool_stream_cqboolagg (k text, b boolean) SERVER pipelinedb; 23 | 24 | CREATE VIEW test_bool_and AS SELECT k::text, bool_and(b::boolean) FROM bool_stream_cqboolagg GROUP BY k; 25 | CREATE VIEW test_bool_or AS SELECT k::text, bool_or(b::boolean) FROM bool_stream_cqboolagg GROUP BY k; 26 | CREATE VIEW test_every AS SELECT k::text, every(b::boolean) FROM bool_stream_cqboolagg GROUP BY k; 27 | 28 | INSERT INTO bool_stream_cqboolagg (k, b) VALUES ('x', 't'), ('x', 't'), ('x', true), ('x', 't'), ('x', 't'), ('x', true), ('x', true); 29 | INSERT INTO bool_stream_cqboolagg (k, b) VALUES ('y', false), ('y', 'f'), ('y', 'f'), ('x', false), ('x', true), ('x', 't'), ('x', 't'); 30 | 31 | SELECT * FROM test_bool_and ORDER BY k DESC; 32 | SELECT * FROM test_bool_or ORDER BY k DESC; 33 | SELECT * FROM test_every ORDER BY k DESC; 34 | 35 | INSERT INTO bool_stream_cqboolagg (k, b) VALUES ('x', 't'), ('x', 'f'); 36 | INSERT INTO bool_stream_cqboolagg (k, b) VALUES ('y', 'f'); 37 | 38 | SELECT * FROM test_bool_and ORDER BY k DESC; 39 | SELECT * FROM test_bool_or ORDER BY k DESC; 40 | SELECT * FROM test_every ORDER BY k DESC; 41 | 42 | DROP FOREIGN TABLE bool_stream_cqboolagg CASCADE; 43 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_view_namespace.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE test_ns_stream (x int) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_cvn0 AS SELECT COUNT(*) FROM test_ns_stream; 4 | CREATE VIEW nonexistent.cv AS SELECT COUNT(*) FROM test_ns_stream; 5 | 6 | CREATE SCHEMA test_cvn_schema0; 7 | CREATE FOREIGN TABLE test_cvn_schema0.test_ns_stream (x int) SERVER pipelinedb; 8 | CREATE VIEW test_cvn_schema0.test_cvn0 AS SELECT COUNT(*) FROM test_cvn_schema0.test_ns_stream; 9 | 10 | CREATE SCHEMA test_cvn_schema1; 11 | CREATE FOREIGN TABLE test_cvn_schema1.test_ns_stream (x int) SERVER pipelinedb; 12 | CREATE VIEW test_cvn_schema1.test_cvn0 AS SELECT COUNT(*) FROM test_cvn_schema1.test_ns_stream; 13 | 14 | SELECT schema, name FROM pipelinedb.get_views() WHERE name LIKE '%test_cvn%' ORDER BY schema, name; 15 | 16 | ALTER SCHEMA test_cvn_schema0 RENAME TO test_cvn_schema0_new; 17 | 18 | CREATE VIEW test_cvn_schema0_new.test_cvn0 AS SELECT COUNT(*) FROM test_cvn_schema0_new.test_ns_stream; 19 | CREATE VIEW test_cvn_schema0_new.test_cvn1 AS SELECT COUNT(*) FROM test_cvn_schema0_new.test_ns_stream; 20 | 21 | SELECT schema, name FROM pipelinedb.get_views() WHERE name LIKE '%test_cvn%' ORDER BY schema, name; 22 | 23 | DROP SCHEMA test_cvn_schema0_new; 24 | DROP SCHEMA test_cvn_schema0_new CASCADE; 25 | 26 | SELECT schema, name FROM pipelinedb.get_views() WHERE name LIKE '%test_cvn%' ORDER BY schema, name; 27 | 28 | DROP VIEW test_cvn0; 29 | DROP SCHEMA test_cvn_schema1 CASCADE; 30 | 31 | SELECT schema, name FROM pipelinedb.get_views() WHERE name LIKE '%test_cvn%' ORDER BY schema, name; 32 | 33 | CREATE FOREIGN TABLE test_cvn_stream (x int) SERVER pipelinedb; 34 | CREATE VIEW test_cvn0 AS SELECT x::int FROM test_cvn_stream; 35 | 36 | CREATE SCHEMA test_cvn_schema0; 37 | CREATE FOREIGN TABLE test_cvn_schema0.test_cvn_stream (x int, y text) SERVER pipelinedb; 38 | CREATE VIEW test_cvn_schema0.test_cvn0 AS SELECT x::int, y::text FROM test_cvn_schema0.test_cvn_stream; 39 | 40 | SELECT schema, name FROM pipelinedb.get_streams() WHERE name='test_cvn_stream' ORDER BY schema; 41 | 42 | INSERT INTO test_cvn_stream (x) VALUES (1); 43 | INSERT INTO test_cvn_schema0.test_cvn_stream (x, y) VALUES (2, 2), (3, 3); 44 | 45 | SELECT * FROM test_cvn0; 46 | SELECT * FROM test_cvn_schema0.test_cvn0 ORDER BY x; 47 | 48 | DROP FOREIGN TABLE test_ns_stream CASCADE; 49 | DROP FOREIGN TABLE test_cvn_stream CASCADE; 50 | DROP SCHEMA test_cvn_schema0 CASCADE; 51 | -------------------------------------------------------------------------------- /src/test/py/test_freq_agg.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | 3 | 4 | def test_freq_agg(pipeline, clean_db): 5 | """ 6 | Test freq_agg, freq_merge_agg 7 | """ 8 | pipeline.create_stream('test_cmsketch_stream', k='int', x='int') 9 | 10 | q = """ 11 | SELECT k::integer, freq_agg(x::int) AS c FROM test_cmsketch_stream 12 | GROUP BY k 13 | """ 14 | desc = ('k', 'x') 15 | pipeline.create_cv('test_cmsketch_agg', q) 16 | 17 | rows = [] 18 | for n in range(1000): 19 | rows.append((0, n % 20)) 20 | rows.append((1, n % 50)) 21 | 22 | pipeline.insert('test_cmsketch_stream', desc, rows) 23 | 24 | result = pipeline.execute('SELECT freq(c, 10) AS x, freq(c, 40) AS y, freq(c, 60) FROM test_cmsketch_agg ORDER BY k') 25 | assert len(result) == 2 26 | assert (result[0][0], result[0][1], result[0][2]) == (50, 0, 0) 27 | assert (result[1][0], result[1][1], result[1][2]) == (20, 20, 0) 28 | 29 | result = pipeline.execute('SELECT freq(combine(c), 10) AS x, freq(combine(c), 40) AS y, freq(combine(c), 60) FROM test_cmsketch_agg') 30 | assert len(result) == 1 31 | (result[0][0], result[0][1], result[0][2]) == (70, 20, 0) 32 | 33 | def test_cmsketch_type(pipeline, clean_db): 34 | pipeline.create_table('test_cmsketch_type', x='int', y='cmsketch') 35 | pipeline.execute('INSERT INTO test_cmsketch_type (x, y) VALUES ' 36 | '(1, cmsketch_empty()), (2, cmsketch_empty())') 37 | 38 | for i in range(1000): 39 | pipeline.execute('UPDATE test_cmsketch_type SET y = freq_add(y, %d %% x)' % i) 40 | 41 | result = list(pipeline.execute('SELECT freq(y, 0), ' 42 | 'freq(y, 1) ' 43 | 'FROM test_cmsketch_type ORDER BY x')) 44 | assert (result[0][0], result[0][1]) == (1000, 0) 45 | assert (result[1][0], result[1][1]) == (500, 500) 46 | 47 | def test_cksketch_frequency(pipeline, clean_db): 48 | pipeline.create_stream('test_cmsketch_stream', k='int', x='int') 49 | 50 | q = """ 51 | SELECT k::integer, freq_agg(x::int) AS c FROM test_cmsketch_stream 52 | GROUP BY k 53 | """ 54 | desc = ('k', 'x') 55 | pipeline.create_cv('test_cmsketch_frequency', q) 56 | 57 | rows = [(n, None) for n in range(100)] 58 | pipeline.insert('test_cmsketch_stream', desc, rows) 59 | 60 | result = pipeline.execute('SELECT freq(c, null) AS x FROM test_cmsketch_frequency ORDER BY k') 61 | assert len(result) == 100 62 | for row in result: 63 | assert row[0] == 0 64 | -------------------------------------------------------------------------------- /src/test/py/test_hs_aggs.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import random 3 | import time 4 | 5 | 6 | def _rank(n, values): 7 | rank = 1 8 | peers = 0 9 | for v in sorted(values): 10 | if v < n: 11 | rank += 1 12 | if v == n: 13 | peers += 1 14 | 15 | return rank, peers 16 | 17 | def _dense_rank(n, values): 18 | return _rank(n, set(values)) 19 | 20 | def _test_hs_agg(pipeline, agg): 21 | values = [random.randint(-100, 100) for n in range(1000)] 22 | h = random.choice(values) + random.randint(-10, 10) 23 | 24 | pipeline.create_stream('stream0', x='int') 25 | cq = 'SELECT %s(%d) WITHIN GROUP (ORDER BY x::integer) FROM stream0' % (agg, h) 26 | pipeline.create_cv('test_%s' % agg, cq) 27 | 28 | pipeline.insert('stream0', ('x',), [(v,) for v in values]) 29 | 30 | result = pipeline.execute('SELECT %s FROM test_%s' % (agg, agg))[0] 31 | 32 | rank, peers = _rank(h, values) 33 | dense_rank, _ = _dense_rank(h, values) 34 | 35 | return rank, dense_rank, peers, len(values), result[agg] 36 | 37 | def test_rank(pipeline, clean_db): 38 | """ 39 | Verify that continuous rank produces the correct result given random input 40 | """ 41 | rank, _, _, _, result = _test_hs_agg(pipeline, 'rank') 42 | 43 | assert rank == result 44 | 45 | def test_dense_rank(pipeline, clean_db): 46 | """ 47 | Verify that continuous dense_rank produces the correct result given random input 48 | """ 49 | _, dense_rank, _, _, result = _test_hs_agg(pipeline, 'dense_rank') 50 | 51 | # We use HLL for dense_rank, so there may be a small margin of error, 52 | # but it should never be larger than 4% 53 | delta = abs(dense_rank - result) 54 | 55 | assert delta / float(dense_rank) <= 0.04 56 | 57 | def test_percent_rank(pipeline, clean_db): 58 | """ 59 | Verify that continuous percent_rank produces the correct result given random input 60 | """ 61 | rank, _, _, row_count, result = _test_hs_agg(pipeline, 'percent_rank') 62 | 63 | percent_rank = (rank - 1) / (float(row_count)) 64 | 65 | assert (percent_rank - result) < 0.0000001 66 | 67 | def test_cume_dist(pipeline, clean_db): 68 | """ 69 | Verify that continuous cume_dist produces the correct result given random input 70 | """ 71 | rank, _, peers, row_count, result = _test_hs_agg(pipeline, 'cume_dist') 72 | 73 | rank += peers 74 | cume_dist = rank / (float(row_count + 1)) 75 | 76 | assert abs(cume_dist - result) < 0.0000001 77 | -------------------------------------------------------------------------------- /src/test/py/extended.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * extended_inserts.c 4 | * 5 | * Writes to streams using the extended protocol for testing 6 | 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | int 17 | main(int argc, char **argv) 18 | { 19 | PGconn *conn; 20 | PGresult *res; 21 | int i; 22 | char connstr[256]; 23 | char *stream; 24 | int nrows; 25 | char *vals[3]; 26 | char buf[1024]; 27 | 28 | srand(time(NULL)); 29 | 30 | if (argc != 5) 31 | { 32 | fprintf(stderr, "Usage: extended <# rows>\n"); 33 | exit(1); 34 | } 35 | 36 | sprintf(connstr, "dbname=%s port=%s host=localhost", argv[1], argv[2]); 37 | stream = argv[3]; 38 | nrows = atoi(argv[4]); 39 | 40 | conn = PQconnectdb(connstr); 41 | if (PQstatus(conn) != CONNECTION_OK) 42 | { 43 | fprintf(stderr, "connection to database failed: %s", PQerrorMessage(conn)); 44 | exit(1); 45 | } 46 | 47 | res = PQexec(conn, "BEGIN"); 48 | if (PQresultStatus(res) != PGRES_COMMAND_OK) 49 | { 50 | fprintf(stderr, "BEGIN command failed: %s", PQerrorMessage(conn)); 51 | exit(1); 52 | } 53 | 54 | PQclear(res); 55 | 56 | sprintf(buf, "INSERT INTO %s (x, y, z) VALUES ($1, $2, $3)", stream); 57 | if ((res = PQprepare(conn, "params", buf, 3, NULL)) == NULL) 58 | fprintf(stderr, "PREPARE command failed: %s", PQerrorMessage(conn)); 59 | 60 | for (i = 0; i < nrows; i++) 61 | { 62 | char x[16]; 63 | char y[16]; 64 | char z[16]; 65 | 66 | sprintf(x, "%d", rand() % 100); 67 | sprintf(y, "%d", rand() % 100); 68 | sprintf(z, "%d", rand() % 100); 69 | 70 | vals[0] = x; 71 | vals[1] = y; 72 | vals[2] = z; 73 | 74 | res = PQexecPrepared(conn, "params", 3, (const char * const *) vals, NULL, NULL, 0); 75 | } 76 | 77 | /* now perform unparamaterized inserts */ 78 | sprintf(buf, "INSERT INTO %s (x, y, z) VALUES (0, 0, 0)", stream); 79 | if ((res = PQprepare(conn, "no params", buf, 0, NULL)) == NULL) 80 | fprintf(stderr, "PREPARE command failed: %s", PQerrorMessage(conn)); 81 | 82 | for (i = 0; i < nrows; i++) 83 | res = PQexecPrepared(conn, "no params", 0, NULL, NULL, NULL, 0); 84 | 85 | res = PQexec(conn, "END"); 86 | PQclear(res); 87 | PQfinish(conn); 88 | 89 | return 0; 90 | } 91 | -------------------------------------------------------------------------------- /src/test/py/test_prepared_inserts.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import getpass 3 | import psycopg2 4 | import random 5 | import subprocess 6 | import time 7 | 8 | 9 | def test_prepared_inserts(pipeline, clean_db): 10 | """ 11 | Verify that we can PREPARE and EXECUTE an INSERT statement 12 | """ 13 | pipeline.create_stream('stream0', x='int', y='float8') 14 | 15 | conn = psycopg2.connect('dbname=postgres user=%s host=localhost port=%s' % (getpass.getuser(), pipeline.port)) 16 | db = conn.cursor() 17 | db.execute('CREATE VIEW test_prepared0 AS SELECT x::integer, COUNT(*), sum(y::integer) FROM stream0 GROUP BY x') 18 | db.execute('CREATE VIEW test_prepared1 AS SELECT x::integer, COUNT(*), sum(y::float8) FROM stream0 GROUP BY x') 19 | conn.commit() 20 | 21 | db.execute('PREPARE ins AS INSERT INTO stream0 (x, y) VALUES ($1, $2)') 22 | 23 | for n in range(10000): 24 | row = (n % 100, random.random()) 25 | db.execute('EXECUTE ins (%s, %s)' % row) 26 | 27 | time.sleep(0.1) 28 | 29 | conn.commit() 30 | 31 | result = pipeline.execute('SELECT * FROM test_prepared0 ORDER BY x') 32 | 33 | assert len(result) == 100 34 | 35 | for n in range(100): 36 | assert result[n]['count'] == 100 37 | 38 | result = pipeline.execute('SELECT * FROM test_prepared1 ORDER BY x') 39 | 40 | assert len(result) == 100 41 | 42 | for n in range(100): 43 | assert result[n]['count'] == 100 44 | 45 | conn.close() 46 | 47 | 48 | def test_prepared_extended(pipeline, clean_db): 49 | """ 50 | Verify that we can write to streams using the extended protocol. This test 51 | shells out to a binary because psycopg2 doesn't use the extended protocol. 52 | """ 53 | pipeline.create_stream('extended_stream', x='int', y='int', z='int') 54 | q = """ 55 | SELECT COUNT(x::integer) AS x, COUNT(y::integer) AS y, COUNT(z::integer) AS z FROM extended_stream 56 | """ 57 | pipeline.create_cv('test_prepared_extended', q) 58 | 59 | # This will insert 1000 via a paramaterized insert, and 1000 via unparamaterized insert 60 | cmd = ['./extended', 'postgres', str(pipeline.port), 'extended_stream', '1000'] 61 | 62 | stdout, stderr = subprocess.Popen(cmd).communicate() 63 | 64 | assert stdout is None 65 | assert stderr is None 66 | 67 | rows = pipeline.execute('SELECT x, y, z FROM test_prepared_extended') 68 | assert len(rows) == 1 69 | 70 | result = rows[0] 71 | 72 | assert result['x'] == 2000 73 | assert result['y'] == 2000 74 | assert result['z'] == 2000 75 | -------------------------------------------------------------------------------- /include/hll.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * hll.h 4 | * Interface for HyperLogLog support 5 | * 6 | * Portions Copyright (c) 2014, Salvatore Sanfilippo 7 | * Portions Copyright (c) 2018, PipelineDB, Inc. 8 | * Portions Copyright (c) 2023, Tantor Labs, Inc. 9 | * 10 | *------------------------------------------------------------------------- 11 | */ 12 | #ifndef PIPELINE_HLL_H 13 | #define PIPELINE_HLL_H 14 | 15 | #include "c.h" 16 | #include "postgres.h" 17 | 18 | #include "executor/tuptable.h" 19 | #include "lib/stringinfo.h" 20 | #include "utils/datum.h" 21 | 22 | #define HLL_MAX_SPARSE_BYTES 11000 23 | #define HLL_MAX_EXPLICIT_REGISTERS 2048 /* 2048 * 4 = 8192 bytes */ 24 | 25 | #define HLL_SPARSE_DIRTY 's' 26 | #define HLL_SPARSE_CLEAN 'S' 27 | #define HLL_DENSE_DIRTY 'd' 28 | #define HLL_DENSE_CLEAN 'D' 29 | #define HLL_EXPLICIT_DIRTY 'e' 30 | #define HLL_EXPLICIT_CLEAN 'E' 31 | #define HLL_UNPACKED 'u' 32 | 33 | #define HLL_IS_SPARSE(hll) ((hll)->encoding == HLL_SPARSE_DIRTY || (hll)->encoding == HLL_SPARSE_CLEAN) 34 | #define HLL_IS_EXPLICIT(hll) ((hll)->encoding == HLL_EXPLICIT_DIRTY || (hll)->encoding == HLL_EXPLICIT_CLEAN) 35 | #define HLL_IS_DENSE(hll) ((hll)->encoding == HLL_DENSE_DIRTY || (hll)->encoding == HLL_DENSE_CLEAN) 36 | #define HLL_IS_UNPACKED(hll) ((hll)->encoding == HLL_UNPACKED) 37 | 38 | #define HLL_EXPLICIT_GET_NUM_REGISTERS(hll) ((hll)->mlen / 4) 39 | 40 | #define HLLSize(hll) (sizeof(HyperLogLog) + (hll)->mlen) 41 | 42 | typedef struct HyperLogLog 43 | { 44 | uint32 vl_len_; 45 | /* Dense or sparse, dirty or clean? See above */ 46 | char encoding; 47 | /* 48 | * Last computed cardinality, can be reused until new data is added. 49 | * That is, if the encoding is *_CLEAN. 50 | */ 51 | long card; 52 | /* number of leading bits of hash values to use for determining register */ 53 | uint8 p; 54 | /* number of bytes allocated for M */ 55 | int mlen; 56 | /* substream registers */ 57 | uint8 M[1]; 58 | } HyperLogLog; 59 | 60 | HyperLogLog *HLLCreateWithP(int p); 61 | HyperLogLog *HLLCreate(void); 62 | HyperLogLog *HLLAdd(HyperLogLog *hll, void *elem, Size len, int *result); 63 | HyperLogLog *HLLCopy(HyperLogLog *src); 64 | uint64 HLLCardinality(HyperLogLog *hll); 65 | HyperLogLog *HLLUnion(HyperLogLog *result, HyperLogLog *incoming); 66 | 67 | HyperLogLog *HLLUnpack(HyperLogLog *initial); 68 | HyperLogLog *HLLPack(HyperLogLog *hllu); 69 | HyperLogLog *HLLUnionAdd(HyperLogLog *hllu, HyperLogLog *incoming); 70 | 71 | #endif 72 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_grouping_sets.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE test_gs_stream (x int, y int, z int) SERVER pipelinedb; 2 | 3 | CREATE VIEW test_gs0 AS SELECT x::integer, y::integer, COUNT(*) FROM test_gs_stream 4 | GROUP BY CUBE (x, y); 5 | 6 | SELECT pipelinedb.get_worker_querydef('test_gs0'); 7 | SELECT pipelinedb.get_combiner_querydef('test_gs0'); 8 | SELECT pg_get_viewdef('test_gs0'); 9 | 10 | CREATE VIEW test_gs1 AS SELECT x::integer, y::integer, z::integer, COUNT(*) FROM test_gs_stream 11 | GROUP BY GROUPING SETS (x), (x, y), (x, y, z), (x, z); 12 | 13 | SELECT pipelinedb.get_worker_querydef('test_gs1'); 14 | SELECT pipelinedb.get_combiner_querydef('test_gs1'); 15 | SELECT pg_get_viewdef('test_gs1'); 16 | 17 | CREATE VIEW test_gs2 AS SELECT x::integer, y::integer, COUNT(*) FROM test_gs_stream 18 | GROUP BY GROUPING SETS (x), (x, y); 19 | 20 | SELECT pipelinedb.get_worker_querydef('test_gs2'); 21 | SELECT pipelinedb.get_combiner_querydef('test_gs2'); 22 | SELECT pg_get_viewdef('test_gs2'); 23 | 24 | CREATE VIEW test_gs3 WITH (sw = '5 seconds') AS SELECT x::integer, y::integer, z::integer, COUNT(*) FROM test_gs_stream 25 | GROUP BY GROUPING SETS (x, y, z); 26 | 27 | SELECT pipelinedb.get_worker_querydef('test_gs3'); 28 | SELECT pipelinedb.get_combiner_querydef('test_gs3'); 29 | SELECT pg_get_viewdef('test_gs3'); 30 | 31 | INSERT INTO test_gs_stream (x, y, z) SELECT x, x * 2 AS y, -x AS z FROM generate_series(1, 100) AS x; 32 | INSERT INTO test_gs_stream (x, y, z) SELECT x, x * 2 AS y, -x AS z FROM generate_series(1, 100) AS x; 33 | INSERT INTO test_gs_stream (x, y, z) SELECT x, x * 2 AS y, -x AS z FROM generate_series(1, 100) AS x; 34 | 35 | SELECT * FROM test_gs0 ORDER BY x, y; 36 | SELECT * FROM test_gs1 ORDER BY x, y, z; 37 | SELECT * FROM test_gs2 ORDER BY x, y; 38 | SELECT * FROM test_gs3 ORDER BY x, y, z; 39 | 40 | INSERT INTO test_gs_stream (x, y, z) SELECT x, x * 2 AS y, -x AS z FROM generate_series(1, 100) AS x; 41 | INSERT INTO test_gs_stream (x, y, z) SELECT x, x * 2 AS y, -x AS z FROM generate_series(1, 100) AS x; 42 | INSERT INTO test_gs_stream (x, y, z) SELECT x, x * 2 AS y, -x AS z FROM generate_series(1, 100) AS x; 43 | 44 | SELECT * FROM test_gs0 ORDER BY x, y; 45 | SELECT * FROM test_gs1 ORDER BY x, y, z; 46 | SELECT * FROM test_gs2 ORDER BY x, y; 47 | SELECT * FROM test_gs3 ORDER BY x, y, z; 48 | 49 | SELECT pg_sleep(5); 50 | SELECT * FROM test_gs3 ORDER BY x, y, z; 51 | 52 | DROP VIEW test_gs0; 53 | DROP VIEW test_gs1; 54 | DROP VIEW test_gs2; 55 | DROP VIEW test_gs3; 56 | 57 | DROP FOREIGN TABLE test_gs_stream CASCADE; 58 | -------------------------------------------------------------------------------- /include/stream_fdw.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * stream_fdw.h 4 | * FDW for PipelineDB streams 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef STREAM_FDW_H 12 | #define STREAM_FDW_H 13 | 14 | #include "executor.h" 15 | #include "executor/tuptable.h" 16 | #include "foreign/foreign.h" 17 | #include "lib/stringinfo.h" 18 | #include "nodes/bitmapset.h" 19 | #include "nodes/execnodes.h" 20 | #include "nodes/plannodes.h" 21 | #include "optimizer/pathnode.h" 22 | #include "microbatch.h" 23 | #include "utils/rel.h" 24 | 25 | #define REENTRANT_STREAM_INSERT 0x10000 26 | 27 | typedef struct StreamProjectionInfo StreamProjectionInfo; 28 | 29 | typedef struct StreamScanState 30 | { 31 | ContExecutor *cont_executor; 32 | StreamProjectionInfo *pi; 33 | Size nbytes; 34 | int ntuples; 35 | } StreamScanState; 36 | 37 | typedef struct StreamInsertState 38 | { 39 | int flags; 40 | Bitmapset *queries; 41 | 42 | int ntups; 43 | long nbytes; 44 | int nbatches; 45 | 46 | microbatch_t *batch; 47 | TupleDesc desc; 48 | 49 | microbatch_ack_t *ack; 50 | uint64 start_generation; 51 | 52 | ContQueryDatabaseMetadata *db_meta; 53 | } StreamInsertState; 54 | 55 | extern int stream_insert_level; 56 | 57 | extern Datum stream_fdw_handler(PG_FUNCTION_ARGS); 58 | 59 | extern void GetStreamSize(PlannerInfo *root, RelOptInfo *baserel, Oid streamid); 60 | extern void GetStreamPaths(PlannerInfo *root, RelOptInfo *baserel, Oid streamid); 61 | extern ForeignScan *GetStreamScanPlan(PlannerInfo *root, RelOptInfo *baserel, 62 | Oid streamid, ForeignPath *best_path, List *tlist, List *scan_clauses, Plan *outer_plan); 63 | extern void BeginStreamScan(ForeignScanState *node, int eflags); 64 | extern List *PlanStreamModify(PlannerInfo *root, ModifyTable *plan, Index resultRelation, int subplan_index); 65 | extern TupleTableSlot *IterateStreamScan(ForeignScanState *node); 66 | extern void ReScanStreamScan(ForeignScanState *node); 67 | extern void EndStreamScan(ForeignScanState *node); 68 | 69 | extern void BeginStreamModify(ModifyTableState *mtstate, ResultRelInfo *resultRelInfo, 70 | List *fdw_private, int subplan_index, int eflags); 71 | extern TupleTableSlot *ExecStreamInsert(EState *estate, ResultRelInfo *resultRelInfo, 72 | TupleTableSlot *slot, TupleTableSlot *planSlot); 73 | extern void EndStreamModify(EState *estate, ResultRelInfo *resultRelInfo); 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_sw_count.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE cqswcount_stream (k text) SERVER pipelinedb; 2 | CREATE VIEW test_count AS SELECT k::text, COUNT(*) FROM cqswcount_stream WHERE arrival_timestamp > clock_timestamp() - interval '60 second' GROUP BY k; 3 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'); 4 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'), ('y'), ('y'), ('y'), ('y'), ('y'), ('y'); 5 | SELECT * FROM test_count ORDER BY k; 6 | k | count 7 | ---+------- 8 | x | 12 9 | y | 6 10 | (2 rows) 11 | 12 | SELECT pg_sleep(1); 13 | pg_sleep 14 | ---------- 15 | 16 | (1 row) 17 | 18 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'); 19 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'), ('x'), ('x'), ('x'), ('x'), ('y'), ('y'), ('y'), ('y'), ('y'), ('y'); 20 | SELECT * FROM test_count ORDER BY k; 21 | k | count 22 | ---+------- 23 | x | 24 24 | y | 12 25 | (2 rows) 26 | 27 | DROP VIEW test_count; 28 | CREATE VIEW sw_count0 AS SELECT COUNT(*) FROM cqswcount_stream WHERE arrival_timestamp > clock_timestamp() - interval '10 second'; 29 | CREATE VIEW sw_count1 AS SELECT combine(count) FROM sw_count0_mrel WHERE arrival_timestamp > clock_timestamp() - interval '5 second'; 30 | INSERT INTO cqswcount_stream (k) VALUES ('x'), ('x'); 31 | SELECT * FROM sw_count0; 32 | count 33 | ------- 34 | 2 35 | (1 row) 36 | 37 | SELECT * FROM sw_count1; 38 | combine 39 | --------- 40 | 2 41 | (1 row) 42 | 43 | SELECT pg_sleep(6); 44 | pg_sleep 45 | ---------- 46 | 47 | (1 row) 48 | 49 | SELECT * FROM sw_count0; 50 | count 51 | ------- 52 | 2 53 | (1 row) 54 | 55 | SELECT * FROM sw_count1; 56 | combine 57 | --------- 58 | 59 | (1 row) 60 | 61 | DROP FOREIGN TABLE cqswcount_stream CASCADE; 62 | NOTICE: drop cascades to 2 other objects 63 | DETAIL: drop cascades to view sw_count0 64 | drop cascades to view sw_count1 65 | -- Verify that we can specify our own sliding-window column 66 | CREATE FOREIGN TABLE sw_col_s (x integer, ts timestamptz) SERVER pipelinedb; 67 | CREATE VIEW sw_col0 WITH (sw = '10 seconds', sw_column = 'ts') AS SELECT COUNT(*) FROM sw_col_s; 68 | INSERT INTO sw_col_s (x, ts) VALUES (0, now() - interval '8 seconds'); 69 | SELECT * FROM sw_col0; 70 | count 71 | ------- 72 | 1 73 | (1 row) 74 | 75 | SELECT pg_sleep(3); 76 | pg_sleep 77 | ---------- 78 | 79 | (1 row) 80 | 81 | SELECT * FROM sw_col0; 82 | count 83 | ------- 84 | 85 | (1 row) 86 | 87 | DROP FOREIGN TABLE sw_col_s CASCADE; 88 | NOTICE: drop cascades to view sw_col0 89 | -------------------------------------------------------------------------------- /src/test/py/test_databases.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import getpass 3 | import psycopg2 4 | from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT 5 | 6 | def test_multiple_databases(pipeline, clean_db): 7 | conn = psycopg2.connect('dbname=postgres user=%s host=localhost port=%s' % (getpass.getuser(), pipeline.port)) 8 | conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT) 9 | 10 | cur = conn.cursor() 11 | cur.execute('CREATE DATABASE tmp_pipeline') 12 | cur.close() 13 | 14 | q = 'SELECT x::int FROM dbstream' 15 | pipeline.create_stream('dbstream', x='int') 16 | pipeline.create_cv('test_multiple_databases', q) 17 | 18 | # Insert data in first database. 19 | pipeline.insert('dbstream', ['x'], map(lambda x: (x,), range(0, 10, 2))) 20 | result = pipeline.execute('SELECT * FROM test_multiple_databases') 21 | assert sorted(row['x'] for row in result) == list(range(0, 10, 2)) 22 | 23 | # Create same CV in the other database, make sure its created and write different data to it. 24 | tmp_conn = psycopg2.connect('dbname=tmp_pipeline user=%s host=localhost port=%s' % (getpass.getuser(), pipeline.port)) 25 | cur = tmp_conn.cursor() 26 | cur.execute('CREATE EXTENSION pipelinedb') 27 | cur.execute('CREATE FOREIGN TABLE dbstream (x int) SERVER pipelinedb') 28 | cur.execute('CREATE VIEW test_multiple_databases AS %s' % q) 29 | tmp_conn.commit() 30 | cur.execute('INSERT INTO dbstream (x) VALUES %s' % ', '.join(map(lambda x: '(%d)' % x, range(1, 11, 2)))) 31 | cur.execute('SELECT * FROM test_multiple_databases') 32 | tmp_conn.commit() 33 | assert sorted(row[0] for row in cur) == list(range(1, 11, 2)) 34 | 35 | # Ensure that the data written to the other database isn't seen by the first database. 36 | result = pipeline.execute('SELECT * FROM test_multiple_databases') 37 | assert sorted(row['x'] for row in result) == list(range(0, 10, 2)) 38 | 39 | # Insert new data to both databases. 40 | pipeline.insert('dbstream', ['x'], map(lambda x: (x,), range(10, 20, 2))) 41 | cur.execute('INSERT INTO dbstream (x) VALUES %s' % ', '.join(map(lambda x: '(%d)' % x, range(11, 21, 2)))) 42 | 43 | # Ensure both databases still saw the data written out to them. 44 | result = pipeline.execute('SELECT * FROM test_multiple_databases') 45 | assert sorted(row['x'] for row in result) == list(range(0, 20, 2)) 46 | cur.execute('SELECT * FROM test_multiple_databases') 47 | tmp_conn.commit() 48 | assert sorted(row[0] for row in cur) == list(range(1, 21, 2)) 49 | 50 | cur.close() 51 | tmp_conn.close() 52 | cur = conn.cursor() 53 | cur.execute('DROP DATABASE tmp_pipeline') 54 | cur.close() 55 | conn.close() 56 | -------------------------------------------------------------------------------- /src/test/regress/expected/date_round.out: -------------------------------------------------------------------------------- 1 | -- nulls 2 | SELECT date_round(NULL, NULL); 3 | date_round 4 | ------------ 5 | 6 | (1 row) 7 | 8 | SELECT date_round(NULL, INTERVAL '5 minute'); 9 | date_round 10 | ------------ 11 | 12 | (1 row) 13 | 14 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', NULL); 15 | date_round 16 | ------------------------------ 17 | Fri Feb 16 20:38:40 2001 PST 18 | (1 row) 19 | 20 | -- valid floors 21 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '15 second'); 22 | date_round 23 | ------------------------------ 24 | Fri Feb 16 20:38:30 2001 PST 25 | (1 row) 26 | 27 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '10 minute'); 28 | date_round 29 | ------------------------------ 30 | Fri Feb 16 20:30:00 2001 PST 31 | (1 row) 32 | 33 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '3 hour'); 34 | date_round 35 | ------------------------------ 36 | Fri Feb 16 19:00:00 2001 PST 37 | (1 row) 38 | 39 | SELECT date_round(TIMESTAMP '2001-02-18 20:38:40', INTERVAL '5 day'); 40 | date_round 41 | ------------------------------ 42 | Sun Feb 18 16:00:00 2001 PST 43 | (1 row) 44 | 45 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '2 month'); 46 | date_round 47 | ------------------------------ 48 | Mon Dec 25 16:00:00 2000 PST 49 | (1 row) 50 | 51 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '7 year'); 52 | date_round 53 | ------------------------------ 54 | Fri Dec 31 16:00:00 1999 PST 55 | (1 row) 56 | 57 | -- invalid floors 58 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '150 second'); 59 | date_round 60 | ------------------------------ 61 | Fri Feb 16 20:37:30 2001 PST 62 | (1 row) 63 | 64 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '65 minute'); 65 | date_round 66 | ------------------------------ 67 | Fri Feb 16 19:45:00 2001 PST 68 | (1 row) 69 | 70 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '48 hour'); 71 | date_round 72 | ------------------------------ 73 | Thu Feb 15 16:00:00 2001 PST 74 | (1 row) 75 | 76 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '36 day'); 77 | date_round 78 | ------------------------------ 79 | Tue Jan 30 16:00:00 2001 PST 80 | (1 row) 81 | 82 | SELECT date_round(TIMESTAMP '2001-02-16 20:38:40', INTERVAL '18 month'); 83 | date_round 84 | ------------------------------ 85 | Fri Dec 31 16:00:00 1999 PST 86 | (1 row) 87 | 88 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_sw_bool_agg.sql: -------------------------------------------------------------------------------- 1 | -- bit_and, bit_or 2 | CREATE FOREIGN TABLE bit_test_sw_sw_stream (k text, b bit) SERVER pipelinedb; 3 | 4 | CREATE VIEW test_sw_bit_and AS SELECT k::text, bit_and(b::bit) FROM bit_test_sw_sw_stream GROUP BY k; 5 | CREATE VIEW test_sw_bit_or AS SELECT k::text, bit_or(b::bit) FROM bit_test_sw_sw_stream WHERE arrival_timestamp > clock_timestamp() - interval '60 second' GROUP BY k; 6 | 7 | INSERT INTO bit_test_sw_sw_stream (k, b) VALUES ('x', 1::bit), ('x', '1'::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit); 8 | INSERT INTO bit_test_sw_sw_stream (k, b) VALUES ('y', 0::bit), ('y', 1::bit), ('y', 0::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit); 9 | 10 | SELECT * FROM test_sw_bit_and ORDER BY k DESC; 11 | SELECT * FROM test_sw_bit_or ORDER BY k DESC; 12 | 13 | SELECT pg_sleep(1); 14 | 15 | INSERT INTO bit_test_sw_sw_stream (k, b) VALUES ('x', 1::bit), ('x', 0::bit); 16 | INSERT INTO bit_test_sw_sw_stream (k, b) VALUES ('y', 0::bit); 17 | 18 | SELECT * FROM test_sw_bit_and ORDER BY k DESC; 19 | SELECT * FROM test_sw_bit_or ORDER BY k DESC; 20 | 21 | DROP FOREIGN TABLE bit_test_sw_sw_stream CASCADE; 22 | 23 | -- bool_and, bool_or, every 24 | CREATE FOREIGN TABLE bool_test_sw_sw_stream (k text, b boolean) SERVER pipelinedb; 25 | 26 | CREATE VIEW test_sw_bool_and AS SELECT k::text, bool_and(b::boolean) FROM bool_test_sw_sw_stream WHERE arrival_timestamp > clock_timestamp() - interval '60 second' GROUP BY k; 27 | CREATE VIEW test_sw_bool_or AS SELECT k::text, bool_or(b::boolean) FROM bool_test_sw_sw_stream WHERE arrival_timestamp > clock_timestamp() - interval '60 second' GROUP BY k; 28 | CREATE VIEW test_sw_every AS SELECT k::text, every(b::boolean) FROM bool_test_sw_sw_stream WHERE arrival_timestamp > clock_timestamp() - interval '60 second' GROUP BY k; 29 | 30 | INSERT INTO bool_test_sw_sw_stream (k, b) VALUES ('x', 't'), ('x', 't'), ('x', true), ('x', 't'), ('x', 't'), ('x', true), ('x', true); 31 | INSERT INTO bool_test_sw_sw_stream (k, b) VALUES ('y', false), ('y', 'f'), ('y', 'f'), ('x', false), ('x', true), ('x', true), ('x', 't'); 32 | 33 | SELECT * FROM test_sw_bool_and ORDER BY k DESC; 34 | SELECT * FROM test_sw_bool_or ORDER BY k DESC; 35 | SELECT * FROM test_sw_every ORDER BY k DESC; 36 | 37 | SELECT pg_sleep(1); 38 | 39 | INSERT INTO bool_test_sw_sw_stream (k, b) VALUES ('x', 't'), ('x', 'f'); 40 | INSERT INTO bool_test_sw_sw_stream (k, b) VALUES ('y', 'f'); 41 | 42 | SELECT * FROM test_sw_bool_and ORDER BY k DESC; 43 | SELECT * FROM test_sw_bool_or ORDER BY k DESC; 44 | SELECT * FROM test_sw_every ORDER BY k DESC; 45 | 46 | DROP FOREIGN TABLE bool_test_sw_sw_stream CASCADE; 47 | -------------------------------------------------------------------------------- /src/test/regress/sql/first_values.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE first_values_t (x int, y int, z int); 2 | CREATE FOREIGN TABLE first_values_s (x int, y int, z int) SERVER pipelinedb; 3 | CREATE VIEW first_values_v AS SELECT x::int, first_values(2, y::int) FROM first_values_s GROUP BY x; 4 | \d first_values_v 5 | 6 | INSERT INTO first_values_t (x, y, z) VALUES (1, 1, 10), (1, 1, 4), (1, 1, 5); 7 | INSERT INTO first_values_t (x, y, z) VALUES (1, 1, -1), (1, 2, 10), (1, 2, 4); 8 | INSERT INTO first_values_t (x, y, z) VALUES (2, 1, 10), (2, NULL, -1), (2, -1, 2); 9 | INSERT INTO first_values_s (x, y, z) VALUES (1, 1, 10), (1, 1, 4), (1, 1, 5); 10 | INSERT INTO first_values_s (x, y, z) VALUES (1, 1, -1), (1, 2, 10), (1, 2, 4); 11 | INSERT INTO first_values_s (x, y, z) VALUES (2, 1, 10), (2, NULL, -1), (2, -1, 2); 12 | 13 | SELECT x, first_values(2, y) FROM first_values_t GROUP BY x ORDER BY x; 14 | SELECT * FROM first_values_v ORDER BY x; 15 | 16 | INSERT INTO first_values_t (x, y, z) VALUES (2, 1, 10), (1, -1, -10), (2, -10, -2); 17 | INSERT INTO first_values_s (x, y, z) VALUES (2, 1, 10), (1, -1, -10), (2, -10, -2); 18 | 19 | SELECT x, first_values(2, y) FROM first_values_t GROUP BY x ORDER BY x; 20 | SELECT * FROM first_values_v ORDER BY x; 21 | 22 | SELECT x, first_values(2, y, z) FROM first_values_t GROUP BY x ORDER BY x; 23 | 24 | DROP VIEW first_values_v; 25 | DROP TABLE first_values_t; 26 | 27 | CREATE VIEW first_values_v_order0 AS SELECT x::int % 10 AS g, first_values(2, x::int, y::int) FROM first_values_s GROUP BY g; 28 | \d first_values_v_order0 29 | 30 | INSERT INTO first_values_s (x, y) SELECT x, x FROM generate_series(500, 1000) AS x; 31 | INSERT INTO first_values_s (x, y) SELECT x, x FROM generate_series(250, 1000) AS x; 32 | INSERT INTO first_values_s (x, y) SELECT x, x FROM generate_series(1, 1000) AS x; 33 | 34 | SELECT * FROM first_values_v_order0 ORDER BY g; 35 | 36 | DROP VIEW first_values_v_order0; 37 | DROP FOREIGN TABLE first_values_s CASCADE; 38 | 39 | CREATE FOREIGN TABLE first_values_s (t0 text, t1 text, t2 text) SERVER pipelinedb; 40 | CREATE VIEW first_values_v_order1 AS 41 | SELECT first_values(3, t0::text, t1::text, t2::text) FROM first_values_s; 42 | 43 | INSERT INTO first_values_s (t0, t1, t2) SELECT x, x, x FROM generate_series(500, 1000) AS x; 44 | INSERT INTO first_values_s (t0, t1, t2) SELECT x, x, x FROM generate_series(250, 1000) AS x; 45 | INSERT INTO first_values_s (t0, t1, t2) SELECT x, x, x FROM generate_series(1, 1000) AS x; 46 | 47 | SELECT * FROM first_values_v_order1; 48 | -- Disabled until #93 49 | -- SELECT combine(first_values) FROM first_values_v_order1; 50 | 51 | DROP VIEW first_values_v_order1; 52 | DROP FOREIGN TABLE first_values_s CASCADE; 53 | -------------------------------------------------------------------------------- /src/test/py/test_typed_streams.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | 3 | 4 | def test_online_add_column(pipeline, clean_db): 5 | """ 6 | Verify that we can add columns to a stream while not affecting running CQs 7 | """ 8 | pipeline.create_stream('stream0', c0='integer') 9 | 10 | pipeline.create_cv('cv0', 'SELECT c0 FROM stream0') 11 | pipeline.insert('stream0', ('c0',), [(n,) for n in range(0, 1000)]) 12 | result = pipeline.execute('SELECT * FROM cv0') 13 | 14 | assert len(result) == 1000 15 | 16 | for row in result: 17 | for col in row: 18 | assert col is not None 19 | 20 | pipeline.execute('ALTER FOREIGN TABLE stream0 ADD c1 integer') 21 | 22 | pipeline.create_cv('cv1', 'SELECT c0, c1 FROM stream0') 23 | pipeline.insert('stream0', ('c0', 'c1'), 24 | [(n, n) for n in range(1000, 2000)]) 25 | result = pipeline.execute('SELECT * FROM cv1 WHERE c1 >= 1000') 26 | 27 | assert len(result) == 1000 28 | 29 | for row in result: 30 | for col in row: 31 | assert col is not None 32 | 33 | pipeline.execute('ALTER FOREIGN TABLE stream0 ADD c2 integer') 34 | pipeline.create_cv('cv2', 'SELECT c0, c1, c2 FROM stream0') 35 | pipeline.insert('stream0', ('c0', 'c1', 'c2'), 36 | [(n, n, n) for n in range(2000, 3000)]) 37 | result = pipeline.execute('SELECT * FROM cv2 WHERE c2 >= 2000') 38 | 39 | assert len(result) == 1000 40 | 41 | for row in result: 42 | for col in row: 43 | assert col is not None 44 | 45 | pipeline.execute('ALTER FOREIGN TABLE stream0 ADD c3 integer') 46 | pipeline.create_cv('cv3', 'SELECT c0, c1, c2, c3 FROM stream0') 47 | pipeline.insert('stream0', ('c0', 'c1', 'c2', 'c3'), 48 | [(n, n, n, n) for n in range(3000, 4000)]) 49 | result = pipeline.execute('SELECT * FROM cv3 WHERE c3 >= 3000') 50 | 51 | assert len(result) == 1000 52 | 53 | for row in result: 54 | for col in row: 55 | assert col is not None 56 | 57 | pipeline.execute('ALTER FOREIGN TABLE stream0 ADD c4 integer') 58 | pipeline.create_cv('cv4', 'SELECT c0, c1, c2, c3, c4 FROM stream0') 59 | pipeline.insert('stream0', ('c0', 'c1', 'c2', 'c3', 'c4'), 60 | [(n, n, n, n, n) for n in range(4000, 5000)]) 61 | result = pipeline.execute('SELECT * FROM cv4 WHERE c4 >= 4000') 62 | 63 | assert len(result) == 1000 64 | 65 | for row in result: 66 | for col in row: 67 | assert col is not None 68 | 69 | def test_online_drop_column(pipeline, clean_db): 70 | pipeline.create_stream('stream1', c0='integer') 71 | 72 | valid = False 73 | try: 74 | pipeline.execute('ALTER FOREIGN TABLE stream1 DROP c0') 75 | valid = True 76 | except: 77 | pass 78 | assert not valid 79 | -------------------------------------------------------------------------------- /src/test/py/test_deadlocks.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import getpass 3 | import psycopg2 4 | import random 5 | import threading 6 | import time 7 | 8 | 9 | def test_concurrent_add_drop(pipeline, clean_db): 10 | """ 11 | Adds and drops continuous views while inserting into a stream so that we 12 | see add/drops in the middle of transactions in workers and combiners. 13 | """ 14 | pipeline.create_stream('stream0', x='int') 15 | q = 'SELECT x::int, COUNT(*) FROM stream0 GROUP BY x' 16 | pipeline.create_cv('cv', q) 17 | 18 | stop = False 19 | values = [(x,) for x in range(10000)] 20 | num_inserted = [0] 21 | 22 | def insert(): 23 | while True: 24 | if stop: 25 | break 26 | pipeline.insert('stream0', ['x'], values) 27 | num_inserted[0] += 1 28 | 29 | def add_drop(prefix): 30 | # Don't share the connection object with the insert thread because we want 31 | # these queries to happen in parallel. 32 | conn = psycopg2.connect('dbname=postgres user=%s host=localhost port=%s' % 33 | (getpass.getuser(), pipeline.port)) 34 | add = 'CREATE VIEW %s AS ' + q 35 | drop = 'DROP VIEW %s' 36 | cur = conn.cursor() 37 | cvs = [] 38 | while True: 39 | if stop: 40 | break 41 | if not cvs: 42 | cv = '%s%s' % (prefix, str(random.random())[2:]) 43 | try: 44 | cur.execute(add % cv) 45 | except psycopg2.errors.DeadlockDetected: 46 | pass 47 | cvs.append(cv) 48 | elif len(cvs) > 10: 49 | cur.execute(drop % cvs.pop()) 50 | else: 51 | r = random.random() 52 | if r > 0.5: 53 | cv = '%s%s' % (prefix, str(r)[2:]) 54 | cur.execute(add % cv) 55 | cvs.append(cv) 56 | else: 57 | try: 58 | cur.execute(drop % cvs.pop()) 59 | except psycopg2.errors.UndefinedTable: 60 | pass 61 | conn.commit() 62 | time.sleep(0.0025) 63 | cur.close() 64 | conn.close() 65 | 66 | threads = [threading.Thread(target=insert), 67 | threading.Thread(target=add_drop, args=('cv1_',)), 68 | threading.Thread(target=add_drop, args=('cv2_',))] 69 | 70 | [t.start() for t in threads] 71 | 72 | time.sleep(10) 73 | stop = True 74 | 75 | [t.join() for t in threads] 76 | 77 | views = pipeline.execute('SELECT name FROM pipelinedb.get_views()') 78 | mrels = pipeline.execute("SELECT relname FROM pg_class WHERE relname LIKE 'cv%%_mrel'") 79 | assert len(views) == len(mrels) 80 | 81 | counts = pipeline.execute('SELECT * FROM cv') 82 | assert len(counts) == 10000 83 | for r in counts: 84 | assert r['count'] == num_inserted[0] 85 | -------------------------------------------------------------------------------- /src/test/py/test_hll.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import random 3 | 4 | 5 | def test_user_low_and_high_card(pipeline, clean_db): 6 | """ 7 | Verify that HLL's with low and high cardinalities are correcly combined 8 | """ 9 | pipeline.create_stream('test_hll_stream', x='int', k='integer') 10 | q = """ 11 | SELECT k::integer, hll_agg(x::integer) FROM test_hll_stream GROUP BY k 12 | """ 13 | desc = ('k', 'x') 14 | pipeline.create_cv('test_hll_agg', q) 15 | 16 | # Low cardinalities 17 | rows = [] 18 | for n in range(1000): 19 | rows.append((0, random.choice((-1, -2)))) 20 | rows.append((1, random.choice((-3, -4)))) 21 | 22 | # High cardinalities 23 | for n in range(10000): 24 | rows.append((2, n)) 25 | rows.append((3, n)) 26 | 27 | pipeline.insert('test_hll_stream', desc, rows) 28 | 29 | result = pipeline.execute('SELECT hll_cardinality(combine(hll_agg)) FROM test_hll_agg WHERE k in (0, 1)')[0] 30 | assert result[0] == 4 31 | 32 | result = pipeline.execute('SELECT hll_cardinality(combine(hll_agg)) FROM test_hll_agg WHERE k in (2, 3)')[0] 33 | assert result[0] == 9976 34 | 35 | result = pipeline.execute('SELECT hll_cardinality(combine(hll_agg)) FROM test_hll_agg')[0] 36 | assert result[0] == 9983 37 | 38 | 39 | def test_hll_agg_hashing(pipeline, clean_db): 40 | """ 41 | Verify that hll_agg correctly hashes different input types 42 | """ 43 | pipeline.create_stream('test_hll_stream', x='int', y='text', z='float8') 44 | q = """ 45 | SELECT hll_agg(x::integer) AS i, 46 | hll_agg(y::text) AS t, 47 | hll_agg(z::float8) AS f FROM test_hll_stream 48 | """ 49 | desc = ('x', 'y', 'z') 50 | pipeline.create_cv('test_hll_hashing', q) 51 | 52 | rows = [] 53 | for n in range(10000): 54 | rows.append((n, '%d' % n, float(n))) 55 | rows.append((n, '%05d' % n, float(n))) 56 | 57 | pipeline.insert('test_hll_stream', desc, rows) 58 | 59 | cvq = """ 60 | SELECT hll_cardinality(i), 61 | hll_cardinality(t), hll_cardinality(f) FROM test_hll_hashing 62 | """ 63 | result = pipeline.execute(cvq) 64 | 65 | assert len(result) == 1 66 | 67 | result = result[0] 68 | 69 | assert result[0] == 9976 70 | assert result[1] == 19951 71 | assert result[2] == 10062 72 | 73 | def test_hll_type(pipeline, clean_db): 74 | pipeline.create_table('test_hll_type', x='int', y='hyperloglog') 75 | pipeline.execute('INSERT INTO test_hll_type (x, y) VALUES ' 76 | '(1, hll_empty()), (2, hll_empty())') 77 | 78 | for i in range(1000): 79 | pipeline.execute('UPDATE test_hll_type SET y = hll_add(y, %d / x)' % i) 80 | 81 | result = pipeline.execute('SELECT hll_cardinality(y) FROM test_hll_type ORDER BY x') 82 | assert result[0][0] == 995 83 | assert result[1][0] == 497 84 | -------------------------------------------------------------------------------- /src/test/regress/expected/stream_insert_subselect.out: -------------------------------------------------------------------------------- 1 | CREATE TABLE stream_subselect_t (x integer); 2 | INSERT INTO stream_subselect_t (SELECT generate_series(1, 100)); 3 | CREATE FOREIGN TABLE stream_subselect_stream (x int, price int, t timestamptz) SERVER pipelinedb; 4 | CREATE VIEW stream_subselect_v0 AS SELECT x::integer FROM stream_subselect_stream; 5 | INSERT INTO stream_subselect_stream (x) (SELECT * FROM stream_subselect_t); 6 | INSERT INTO stream_subselect_stream (x) (SELECT * FROM (SELECT x AS y FROM stream_subselect_t) s0); 7 | SELECT COUNT(DISTINCT x) FROM stream_subselect_v0; 8 | count 9 | ------- 10 | 100 11 | (1 row) 12 | 13 | CREATE VIEW stream_subselect_v1 AS SELECT x::integer, COUNT(*) FROM stream_subselect_stream GROUP BY x; 14 | INSERT INTO stream_subselect_stream (x) (SELECT generate_series(1, 1000)); 15 | SELECT COUNT(DISTINCT x) FROM stream_subselect_v1; 16 | count 17 | ------- 18 | 1000 19 | (1 row) 20 | 21 | -- It's not possible to SELECT from another stream in a stream INSERT 22 | CREATE FOREIGN TABLE stream_subselect_s0 (x integer) SERVER pipelinedb; 23 | INSERT INTO stream_subselect_stream (x) (SELECT x FROM stream_subselect_s0); 24 | ERROR: "stream_subselect_s0" is a stream 25 | HINT: Streams can only be read by a continuous view's FROM clause. 26 | CREATE VIEW stream_subselect_v2 AS SELECT x::integer FROM stream_subselect_stream; 27 | INSERT INTO stream_subselect_stream (x) (SELECT generate_series(1, 1000) AS x ORDER BY random()); 28 | SELECT COUNT(DISTINCT x) FROM stream_subselect_v2; 29 | count 30 | ------- 31 | 1000 32 | (1 row) 33 | 34 | CREATE VIEW stream_subselect_v3 AS SELECT x::integer FROM stream_subselect_stream; 35 | INSERT INTO stream_subselect_stream (x) (SELECT * FROM stream_subselect_t WHERE x IN (SELECT generate_series(1, 20))); 36 | SELECT * FROM stream_subselect_v3 ORDER BY x; 37 | x 38 | ---- 39 | 1 40 | 2 41 | 3 42 | 4 43 | 5 44 | 6 45 | 7 46 | 8 47 | 9 48 | 10 49 | 11 50 | 12 51 | 13 52 | 14 53 | 15 54 | 16 55 | 17 56 | 18 57 | 19 58 | 20 59 | (20 rows) 60 | 61 | CREATE VIEW stream_subselect_v4 AS SELECT COUNT(*) FROM stream_subselect_stream; 62 | INSERT INTO stream_subselect_stream (price, t) SELECT 10 + random() AS price, current_timestamp - interval '1 minute' * random() AS t FROM generate_series(1, 1000); 63 | SELECT * FROM stream_subselect_v4; 64 | count 65 | ------- 66 | 1000 67 | (1 row) 68 | 69 | DROP FOREIGN TABLE stream_subselect_stream CASCADE; 70 | NOTICE: drop cascades to 5 other objects 71 | DETAIL: drop cascades to view stream_subselect_v0 72 | drop cascades to view stream_subselect_v1 73 | drop cascades to view stream_subselect_v2 74 | drop cascades to view stream_subselect_v3 75 | drop cascades to view stream_subselect_v4 76 | DROP TABLE stream_subselect_t; 77 | -------------------------------------------------------------------------------- /include/fss.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * fss.h 4 | * Interface for Filtered Space Saving support 5 | * 6 | * Copyright (c) 2023, Tantor Labs, Inc. 7 | * Copyright (c) 2018, PipelineDB, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | #ifndef PIPELINE_FSS_H 12 | #define PIPELINE_FSS_H 13 | 14 | #include "c.h" 15 | #include "postgres.h" 16 | #include "utils/array.h" 17 | #include "utils/typcache.h" 18 | 19 | #define FSS_STORES_DATUMS(fss) ((fss)->top_k != NULL) 20 | 21 | #define ELEMENT_SET 0x01 22 | #define ELEMENT_NEW 0x02 23 | #define ELEMENT_NULL 0x04 24 | 25 | #define IS_SET(el) ((el)->flags & ELEMENT_SET) 26 | #define IS_NEW(el) ((el)->flags & ELEMENT_NEW) 27 | #define IS_NULL(el) ((el)->flags & ELEMENT_NULL) 28 | #define SET_NEW(el) ((el)->flags |= (ELEMENT_SET | ELEMENT_NEW)) 29 | #define UNSET_NEW(el) ((el)->flags = ELEMENT_SET) 30 | #define SET(el) ((el)->flags |= ELEMENT_SET) 31 | #define SET_NULL(el) ((el)-> flags |= ELEMENT_NULL) 32 | 33 | typedef struct FSSTypeInfo 34 | { 35 | Oid typoid; 36 | int16 typlen; 37 | bool typbyval; 38 | char typalign; 39 | char typtype; 40 | } FSSTypeInfo; 41 | 42 | typedef struct Counter 43 | { 44 | uint64_t alpha; 45 | uint16_t count; 46 | } Counter; 47 | 48 | typedef struct MonitoredElement 49 | { 50 | char flags; 51 | uint64_t frequency; 52 | uint32_t error; 53 | uint16_t counter; 54 | uint16_t varlen_index; 55 | Datum value; 56 | } MonitoredElement; 57 | 58 | typedef struct FSS 59 | { 60 | uint32 vl_len_; 61 | uint64_t count; 62 | uint16_t h; 63 | uint16_t m; 64 | uint16_t k; 65 | FSSTypeInfo typ; 66 | Counter *bitmap_counter; /* length h */ 67 | MonitoredElement *monitored_elements; /* length m */ 68 | ArrayType *top_k; /* length k */ 69 | } FSS; 70 | 71 | extern FSS *FSSFromBytes(struct varlena *bytes); 72 | extern FSS *FSSCreateWithMAndH(uint16_t k, TypeCacheEntry *typ, uint16_t m, uint16_t h); 73 | extern FSS *FSSCreate(uint64_t k, TypeCacheEntry *typ); 74 | extern void FSSDestroy(FSS *fss); 75 | 76 | extern uint64_t HashDatum(FSS* fss, Datum d); 77 | extern int MonitoredElementComparator(const void *a, const void *b); 78 | 79 | extern FSS *FSSCopy(FSS *fss); 80 | extern FSS *FSSIncrement(FSS *fss, Datum datum, bool isnull); 81 | extern FSS *FSSIncrementWeighted(FSS *fss, Datum datum, bool isnull, uint64_t weight); 82 | extern FSS *FSSMerge(FSS *fss, FSS *incoming); 83 | extern int FSSMonitoredLength(FSS *fss); 84 | extern Datum *FSSTopK(FSS *fss, uint16_t k, bool **nulls, uint16_t *found); 85 | extern uint64_t *FSSTopKCounts(FSS *fss, uint16_t k, uint16_t *found); 86 | extern uint64_t FSSTotal(FSS *fss); 87 | extern Size FSSSize(FSS *fss); 88 | extern FSS *FSSCompress(FSS *fss); 89 | extern void FSSPrint(FSS *fss); 90 | 91 | #endif 92 | -------------------------------------------------------------------------------- /src/test/py/test_pipeline_funcs.py: -------------------------------------------------------------------------------- 1 | from base import async_insert, pipeline, clean_db 2 | import getpass 3 | import psycopg2 4 | import threading 5 | import time 6 | 7 | 8 | def test_combine_table(pipeline, clean_db): 9 | pipeline.create_stream('s', x='int') 10 | pipeline.create_cv('combine_table', 11 | 'SELECT x::int, COUNT(*) FROM s GROUP BY x') 12 | 13 | values = [(i,) for i in range(1000)] 14 | pipeline.insert('s', ('x',), values) 15 | 16 | pipeline.execute('SELECT * INTO tmprel FROM combine_table_mrel') 17 | 18 | stop = False 19 | ninserts = [0] 20 | 21 | def insert(): 22 | while not stop: 23 | pipeline.insert('s', ('x',), values) 24 | ninserts[0] += 1 25 | time.sleep(0.01) 26 | 27 | t = threading.Thread(target=insert) 28 | t.start() 29 | 30 | time.sleep(2) 31 | 32 | conn = psycopg2.connect('dbname=postgres user=%s host=localhost port=%s' % 33 | (getpass.getuser(), pipeline.port)) 34 | cur = conn.cursor() 35 | cur.execute("SELECT pipelinedb.combine_table('combine_table', 'tmprel')") 36 | conn.commit() 37 | conn.close() 38 | 39 | stop = True 40 | t.join() 41 | 42 | assert ninserts[0] > 0 43 | 44 | rows = list(pipeline.execute('SELECT count FROM combine_table')) 45 | assert len(rows) == 1000 46 | for row in rows: 47 | assert row[0] == ninserts[0] + 2 48 | 49 | 50 | def test_combine_table_no_groups(pipeline, clean_db): 51 | pipeline.create_stream('s', x='int') 52 | pipeline.create_cv('no_groups', 'SELECT COUNT(*) FROM s') 53 | values = [(i,) for i in range(1000)] 54 | pipeline.insert('s', ('x',), values) 55 | 56 | pipeline.execute('SELECT * INTO tmprel FROM no_groups_mrel') 57 | pipeline.execute("SELECT pipelinedb.combine_table('no_groups', 'tmprel')") 58 | 59 | rows = pipeline.execute('SELECT count FROM no_groups') 60 | assert len(rows) == 1 61 | assert len(rows[0]) == 2 62 | assert rows[0][0] == 2000 63 | 64 | 65 | def test_pipeline_flush(pipeline, clean_db): 66 | pipeline.execute('SET pipelinedb.stream_insert_level=async') 67 | pipeline.create_stream('s', x='int') 68 | 69 | pipeline.create_cv('flush', 'SELECT x, cq_sleep(0.01) FROM s') 70 | 71 | values = [(i,) for i in range(1000)] 72 | start = time.time() 73 | 74 | # This will take 0.01 * 1000 = 10s to process but return immediately since 75 | # inserts are async and values will fit in one batch. 76 | pipeline.insert('s', ('x',), values) 77 | insert_end = time.time() 78 | 79 | pipeline.execute('SELECT pipelinedb.flush()') 80 | flush_end = time.time() 81 | 82 | assert insert_end - start < 0.1 83 | assert flush_end - start > 10 84 | 85 | row = list(pipeline.execute('SELECT count(*) FROM flush'))[0] 86 | assert row[0] == 1000 87 | 88 | pipeline.execute('SET pipelinedb.stream_insert_level=sync_commit') 89 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | MODULE_big = pipelinedb 2 | EXTVERSION = $(shell grep default_version $(MODULE_big).control | \ 3 | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") 4 | EXTREVISION = $(shell git rev-parse --short HEAD 2>/dev/null || \ 5 | grep commit_sha $(MODULE_big).control | \ 6 | sed -e "s/commit_sha[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") 7 | 8 | PG_CPPFLAGS += -DPIPELINE_VERSION_STR=\"$(EXTVERSION)\" 9 | PG_CPPFLAGS += -DPIPELINE_REVISION_STR=\"$(EXTREVISION)\" 10 | 11 | SOURCES = $(shell find src -type f -name '*.c' -not -path 'src/test/*') 12 | OBJS = $(patsubst %.c,%.o,$(SOURCES)) 13 | 14 | PG_CONFIG := pg_config 15 | 16 | EXTENSION = pipelinedb 17 | REGRESS = $(EXTENSION) 18 | # TODO: Parallel tests fail randomly. Needs to be investigated. For now, we 19 | # use serial_schedule instead of parallel_schedule 20 | REGRESS_OPTS = --schedule=./src/test/regress/serial_schedule \ 21 | --host=localhost \ 22 | --inputdir=./src/test/regress \ 23 | --outputdir=./src/test/regress \ 24 | --load-extension=pipelinedb \ 25 | --temp-config=./src/test/regress/pipelinedb.conf \ 26 | --bindir=$(bindir) \ 27 | --temp-instance=./src/test/tmp/pgsql 28 | 29 | DATA = $(shell find . -type f -name 'pipelinedb--*.sql') 30 | EXTRA_CLEAN = src/test/regress/expected/$(REGRESS).out src/test/regress/sql/$(REGRESS).sql 31 | SHLIB_LINK += -lzmq -lstdc++ 32 | 33 | ifdef USE_PGXS 34 | PG_CPPFLAGS += -I./include -I$(shell $(PG_CONFIG) --includedir) 35 | PG_CFLAGS += -ggdb 36 | 37 | PGXS := $(shell $(PG_CONFIG) --pgxs) 38 | include $(PGXS) 39 | else 40 | 41 | $(shell touch src/test/regress/sql/$(REGRESS).sql) 42 | $(shell touch src/test/regress/expected/$(REGRESS).out) 43 | 44 | bindir = $(shell $(PG_CONFIG) --bindir) 45 | REGRESS_OPTS = --schedule=./src/test/regress/parallel_schedule \ 46 | --host=localhost \ 47 | --inputdir=./src/test/regress \ 48 | --outputdir=./src/test/regress \ 49 | --load-extension=pipelinedb \ 50 | --temp-config=./src/test/regress/pipelinedb.conf \ 51 | --bindir=$(bindir) 52 | 53 | NO_GENERATED_HEADERS = 1 54 | NO_PGXS = 1 55 | NO_TEMP_INSTALL = 1 56 | top_builddir = $(shell $(PG_CONFIG) --pkglibdir)/pgxs 57 | 58 | include $(shell $(PG_CONFIG) --pkglibdir)/pgxs/src/Makefile.global 59 | include $(shell $(PG_CONFIG) --pgxs) 60 | 61 | endif 62 | 63 | bin_dir = ./bin 64 | 65 | headers_dir = $(shell $(PG_CONFIG) --includedir-server)/../pipelinedb 66 | 67 | # Headers for other extensions to build against 68 | install-headers: 69 | $(MKDIR_P) $(headers_dir) 70 | $(INSTALL_DATA) $(CURDIR)/include/*.h '$(headers_dir)' 71 | 72 | install: install-headers 73 | 74 | bootstrap: 75 | $(bin_dir)/bootstrap 76 | 77 | run: 78 | $(bin_dir)/run-dev 79 | 80 | test: 81 | $(MKDIR_P) $(CURDIR)/src/test/tmp 82 | ifdef USE_PGXS 83 | make install 84 | make installcheck 85 | else 86 | make check 87 | endif 88 | make -C $(CURDIR)/src/test/py test 89 | -------------------------------------------------------------------------------- /src/test/regress/expected/stream_exprs.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE test_exprs_stream (b boolean, t text, n numeric) SERVER pipelinedb; 2 | CREATE VIEW test_stream_exprs AS SELECT b::boolean, t::text, n::numeric FROM test_exprs_stream; 3 | INSERT INTO test_exprs_stream (b, t, n) VALUES (true and true, substring('string!', 1, 3), 1.2 + 100.0001); 4 | INSERT INTO test_exprs_stream (b, t, n) VALUES (1 < 2, 'first' || 'second', 100 % 2 * log(2, 3)); 5 | INSERT INTO test_exprs_stream (b, t, n) VALUES (1 is null, lower('UPPER'), pow(factorial(6)::float8 + 4, 3)); 6 | INSERT INTO test_exprs_stream (b, t, n) VALUES (true and true and true and false, trim(both 'x' from 'xTomxx'), pi() * mod(1 + 3, 4 * 8)); 7 | INSERT INTO test_exprs_stream (b, t, n) VALUES ('t', overlay('Txxxxas' placing 'hom' from 2 for 4), acos(1) * trunc(4.003)); 8 | INSERT INTO test_exprs_stream (b, t, n) VALUES ('on' and true, substring(upper('lower'), 1, 2), 42.82 - @(-12)); 9 | INSERT INTO test_exprs_stream (b, t, n) VALUES ('on' or 'off', btrim('xyxtrimyyx', 'xy'), 2^32); 10 | INSERT INTO test_exprs_stream (b, t, n) VALUES (1 * 2 > 2 * 3, md5(md5(md5('md5 me three times!'))), |/(64)); 11 | INSERT INTO test_exprs_stream (b, t, n) VALUES (1 + 4 * 3 - 14 < 1.2, 'true', pow(2::float8, 32::numeric)); 12 | INSERT INTO test_exprs_stream (b, t, n) VALUES (true and not false or true, substring('left', 1, 2) || substring('right', 1, 2), 1 << 4 << 4); 13 | INSERT INTO test_exprs_stream (b, t, n) VALUES (null and true, substring('x' || 'blah' || 'x', 2, 4), (100 + 24) * (16 * 3 + 4) - (16 << 2 >> 2)); 14 | -- Nonexistent fields should default to null 15 | INSERT INTO test_exprs_stream (b) VALUES (false); 16 | INSERT INTO test_exprs_stream (t) VALUES ('text!'); 17 | INSERT INTO test_exprs_stream (n) VALUES (1 - 1); 18 | SELECT * FROM test_stream_exprs ORDER BY b, t, n; 19 | b | t | n 20 | ---+----------------------------------+-------------------- 21 | f | 497de41c131f736ed205358514b1e954 | 8 22 | f | Tom | 12.5663706143592 23 | f | upper | 379503424 24 | f | | 25 | t | LO | 30.82 26 | t | Thomas | 0 27 | t | firstsecond | 0.0000000000000000 28 | t | leri | 256 29 | t | str | 101.2001 30 | t | trim | 4294967296 31 | t | true | 4294967296 32 | | blah | 6432 33 | | text! | 34 | | | 0 35 | (14 rows) 36 | 37 | DROP FOREIGN TABLE test_exprs_stream CASCADE; 38 | NOTICE: drop cascades to view test_stream_exprs 39 | -------------------------------------------------------------------------------- /include/executor.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * executor.h 4 | * 5 | * Copyright (c) 2023, Tantor Labs, Inc. 6 | * Copyright (c) 2018, PipelineDB, Inc. 7 | * 8 | * ------------------------------------------------------------------------- 9 | */ 10 | #ifndef PIPELINEDB_EXECUTOR 11 | #define PIPELINEDB_EXECUTOR 12 | 13 | #include "postgres.h" 14 | 15 | #include "nodes/execnodes.h" 16 | #include "pipeline_query.h" 17 | #include "reader.h" 18 | #include "scheduler.h" 19 | #include "stats.h" 20 | #include "storage/lockdefs.h" 21 | #include "tcop/dest.h" 22 | #include "utils/relcache.h" 23 | #include "utils/tuplestore.h" 24 | 25 | extern Oid PipelineExecLockRelationOid; 26 | 27 | extern MemoryContext ContQueryTransactionContext; 28 | extern MemoryContext ContQueryBatchContext; 29 | 30 | typedef struct ContQueryState 31 | { 32 | Oid query_id; 33 | ContQuery *query; 34 | MemoryContext state_cxt; 35 | MemoryContext tmp_cxt; 36 | ProcStatsEntry *stats; 37 | } ContQueryState; 38 | 39 | typedef struct BatchReceiver 40 | { 41 | Tuplestorestate *buffer; 42 | void (*flush) (struct BatchReceiver *self, TupleTableSlot *slot); 43 | } BatchReceiver; 44 | 45 | typedef struct ContExecutor ContExecutor; 46 | typedef ContQueryState *(*ContQueryStateInit) (ContExecutor *exec, ContQueryState *state); 47 | typedef Relation ContExecutionLock; 48 | 49 | struct ContExecutor 50 | { 51 | MemoryContext cxt; 52 | 53 | ContQueryProcType ptype; 54 | char *pname; 55 | 56 | Bitmapset *all_queries; 57 | Bitmapset *exec_queries; 58 | 59 | ipc_tuple_reader_batch *batch; 60 | 61 | Oid curr_query_id; 62 | int bms_id; 63 | 64 | ContQueryState *curr_query; 65 | ContQueryState *states[MAX_CQS]; 66 | ContQueryStateInit initfn; 67 | ContExecutionLock lock; 68 | }; 69 | 70 | extern ContExecutionLock AcquireContExecutionLock(LOCKMODE mode); 71 | extern void ReleaseContExecutionLock(ContExecutionLock lock); 72 | 73 | extern void ContExecutorCommit(ContExecutor *exec); 74 | extern void ContExecutorBegin(ContExecutor *exec); 75 | extern ContExecutor *ContExecutorNew(ContQueryStateInit initfn); 76 | extern void ContExecutorDestroy(ContExecutor *exec); 77 | extern void ContExecutorStartBatch(ContExecutor *exec, int timeout); 78 | extern Oid ContExecutorStartNextQuery(ContExecutor *exec, int timeout); 79 | extern void ContExecutorPurgeQuery(ContExecutor *exec); 80 | extern void *ContExecutorIterate(ContExecutor *exec, int *len); 81 | extern void ContExecutorEndQuery(ContExecutor *exec); 82 | extern void ContExecutorEndBatch(ContExecutor *exec, bool commit); 83 | extern void ContExecutorAbortQuery(ContExecutor *exec); 84 | 85 | extern void ExecuteContPlan(EState *estate, PlanState *planstate, 86 | bool use_parallel_mode, 87 | CmdType operation, 88 | bool sendTuples, 89 | uint64 numberTuples, 90 | ScanDirection direction, 91 | DestReceiver *dest, 92 | bool execute_once); 93 | 94 | #endif 95 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_pk.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE test_pk_stream (x int) SERVER pipelinedb; 2 | CREATE VIEW test_pk0 WITH (pk='x') AS SELECT x::integer, COUNT(*) FROM test_pk_stream GROUP BY x; 3 | \d+ test_pk0_mrel 4 | Table "public.test_pk0_mrel" 5 | Column | Type | Collation | Nullable | Default | Storage | Stats target | Description 6 | --------+---------+-----------+----------+---------+---------+--------------+------------- 7 | x | integer | | not null | | plain | | 8 | count | bigint | | | | plain | | 9 | Indexes: 10 | "test_pk0_mrel_pkey" PRIMARY KEY, btree (x) WITH (fillfactor='50') 11 | "test_pk0_mrel_expr_idx" btree (pipelinedb.hash_group(x)) WITH (fillfactor='50') 12 | Options: fillfactor=50 13 | 14 | INSERT INTO test_pk_stream (x) SELECT generate_series(1, 20) AS x; 15 | INSERT INTO test_pk_stream (x) SELECT generate_series(1, 30) AS x; 16 | SELECT * FROM test_pk0 ORDER BY x; 17 | x | count 18 | ----+------- 19 | 1 | 2 20 | 2 | 2 21 | 3 | 2 22 | 4 | 2 23 | 5 | 2 24 | 6 | 2 25 | 7 | 2 26 | 8 | 2 27 | 9 | 2 28 | 10 | 2 29 | 11 | 2 30 | 12 | 2 31 | 13 | 2 32 | 14 | 2 33 | 15 | 2 34 | 16 | 2 35 | 17 | 2 36 | 18 | 2 37 | 19 | 2 38 | 20 | 2 39 | 21 | 1 40 | 22 | 1 41 | 23 | 1 42 | 24 | 1 43 | 25 | 1 44 | 26 | 1 45 | 27 | 1 46 | 28 | 1 47 | 29 | 1 48 | 30 | 1 49 | (30 rows) 50 | 51 | DROP VIEW test_pk0; 52 | CREATE VIEW test_pk1 WITH (pk='count') AS SELECT x::integer, COUNT(*) FROM test_pk_stream GROUP BY x; 53 | \d+ test_pk1_mrel 54 | Table "public.test_pk1_mrel" 55 | Column | Type | Collation | Nullable | Default | Storage | Stats target | Description 56 | --------+---------+-----------+----------+---------+---------+--------------+------------- 57 | x | integer | | | | plain | | 58 | count | bigint | | not null | | plain | | 59 | Indexes: 60 | "test_pk1_mrel_pkey" PRIMARY KEY, btree (count) WITH (fillfactor='50') 61 | "test_pk1_mrel_expr_idx" btree (pipelinedb.hash_group(x)) WITH (fillfactor='50') 62 | Options: fillfactor=50 63 | 64 | INSERT INTO test_pk_stream (x) VALUES (0); 65 | INSERT INTO test_pk_stream (x) VALUES (0); 66 | INSERT INTO test_pk_stream (x) VALUES (1); 67 | INSERT INTO test_pk_stream (x) VALUES (1); 68 | SELECT * FROM test_pk1 ORDER BY x; 69 | x | count 70 | ---+------- 71 | 0 | 1 72 | (1 row) 73 | 74 | DROP VIEW test_pk1; 75 | CREATE VIEW wrong_arg_type WITH (pk=1) AS SELECT COUNT(*) FROM test_pk_stream; 76 | ERROR: continuous view primary keys must be specified with a valid column name 77 | CREATE VIEW no_column WITH (pk='not_here') AS SELECT COUNT(*) FROM test_pk_stream; 78 | ERROR: primary key column "not_here" not found 79 | DROP FOREIGN TABLE test_pk_stream CASCADE; 80 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_part_fillfactor.out: -------------------------------------------------------------------------------- 1 | drop function if exists list_partitions(name); 2 | NOTICE: function list_partitions(name) does not exist, skipping 3 | create function 4 | list_partitions( 5 | baserel name 6 | ) 7 | returns table ( 8 | parent name, child text, options text[] 9 | ) 10 | as $BODY$ 11 | begin 12 | return query select 13 | parent.relname as parent, 14 | regexp_replace(child.relname, 'mrel_[0-9]+_', 'mrel_OID_') as child, 15 | child.reloptions as options 16 | from pg_inherits 17 | join pg_class parent on pg_inherits.inhparent = parent.oid 18 | join pg_class child on pg_inherits.inhrelid = child.oid 19 | join pg_namespace nmsp_parent on nmsp_parent.oid = parent.relnamespace 20 | join pg_namespace nmsp_child on nmsp_child.oid = child.relnamespace 21 | where 22 | parent.relname = baserel || '_mrel' 23 | order by child 24 | ; 25 | end; 26 | $BODY$ 27 | language plpgsql; 28 | create foreign table part_ff_stream (ts timestamp, x int) server pipelinedb; 29 | create view cv_part_ff 30 | with (partition_by=ts, partition_duration='1 hour') 31 | as select ts, count(*) from part_ff_stream group by ts; 32 | \d+ cv_part_ff_mrel; 33 | Partitioned table "public.cv_part_ff_mrel" 34 | Column | Type | Collation | Nullable | Default | Storage | Stats target | Description 35 | --------+-----------------------------+-----------+----------+---------+---------+--------------+------------- 36 | ts | timestamp without time zone | | not null | | plain | | 37 | count | bigint | | | | plain | | 38 | $pk | bigint | | not null | | plain | | 39 | Partition key: RANGE (ts) 40 | Indexes: 41 | "cv_part_ff_mrel_pkey" PRIMARY KEY, btree ("$pk", ts) 42 | "cv_part_ff_mrel_expr_idx" btree (pipelinedb.ls_hash_group(ts)) 43 | Number of partitions: 0 44 | 45 | insert into part_ff_stream values ('2024-11-12 10:18:42', 1); 46 | select * from list_partitions('cv_part_ff'); 47 | parent | child | options 48 | -----------------+----------------------------------------+----------------- 49 | cv_part_ff_mrel | mrel_OID_20241112100000_20241112110000 | {fillfactor=50} 50 | (1 row) 51 | 52 | insert into part_ff_stream values ('2024-11-12 20:18:42', 2); 53 | select * from list_partitions('cv_part_ff'); 54 | parent | child | options 55 | -----------------+----------------------------------------+----------------- 56 | cv_part_ff_mrel | mrel_OID_20241112100000_20241112110000 | {fillfactor=50} 57 | cv_part_ff_mrel | mrel_OID_20241112200000_20241112210000 | {fillfactor=50} 58 | (2 rows) 59 | 60 | drop foreign table part_ff_stream cascade; 61 | NOTICE: drop cascades to view cv_part_ff 62 | -------------------------------------------------------------------------------- /src/test/regress/expected/hll.out: -------------------------------------------------------------------------------- 1 | SELECT hll_cardinality(hll_agg('xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' || x)) FROM generate_series(1, 100) AS x; 2 | hll_cardinality 3 | ----------------- 4 | 100 5 | (1 row) 6 | 7 | SELECT hll_cardinality(hll_agg(x * 1.1)) FROM generate_series(1, 100) AS x; 8 | hll_cardinality 9 | ----------------- 10 | 99 11 | (1 row) 12 | 13 | SELECT hll_cardinality(hll_agg(x::numeric)) FROM generate_series(1, 100) AS x; 14 | hll_cardinality 15 | ----------------- 16 | 100 17 | (1 row) 18 | 19 | SELECT hll_cardinality(hll_agg(x)) FROM generate_series(1, 10000) AS x; 20 | hll_cardinality 21 | ----------------- 22 | 9976 23 | (1 row) 24 | 25 | SELECT hll_cardinality(hll_agg(x)) FROM generate_series(1, 100000) AS x; 26 | hll_cardinality 27 | ----------------- 28 | 99703 29 | (1 row) 30 | 31 | SELECT hll_cardinality(hll_union_agg(hll_agg)) FROM (SELECT hll_agg(x) FROM generate_series(1, 100) AS x UNION ALL SELECT hll_agg(x) FROM generate_series(101, 200) AS x) _; 32 | hll_cardinality 33 | ----------------- 34 | 200 35 | (1 row) 36 | 37 | SELECT hll_cardinality(hll_union_agg(hll_agg)) FROM (SELECT hll_agg(x) FROM generate_series(1, 100) AS x UNION ALL SELECT hll_agg(x) FROM generate_series(101, 2000) AS x) _; 38 | hll_cardinality 39 | ----------------- 40 | 1996 41 | (1 row) 42 | 43 | SELECT hll_cardinality( 44 | hll_union( 45 | (SELECT hll_agg(x) FROM generate_series(1, 1000) AS x), 46 | (SELECT hll_agg(x) FROM generate_series(100, 1100) AS x), 47 | NULL)); 48 | hll_cardinality 49 | ----------------- 50 | 1094 51 | (1 row) 52 | 53 | -- Different HLL dimensions 54 | SELECT hll_cardinality( 55 | hll_union( 56 | (SELECT hll_agg(x, 12) FROM generate_series(1, 1000) AS x), 57 | (SELECT hll_agg(x) FROM generate_series(100, 1100) AS x))); 58 | ERROR: hyperloglogs must have the same p 59 | -- Wrong type 60 | SELECT hll_cardinality( 61 | hll_union( 62 | 'not an hll', 63 | (SELECT hll_agg(x) FROM generate_series(100, 1100) AS x))); 64 | ERROR: argument 1 is not of type "hyperloglog" 65 | -- Byref but not varlena types 66 | CREATE TABLE byref (uuid uuid, name name); 67 | INSERT INTO byref (uuid, name) VALUES ('fe636e28-db82-43af-ac48-df26a4cda1f3', 'alice'); 68 | INSERT INTO byref (uuid, name) VALUES ('fe636e28-db82-43af-ac48-df26a4cda1f3', 'alice'); 69 | SELECT hll_cardinality(hll_agg(uuid)) FROM byref; 70 | hll_cardinality 71 | ----------------- 72 | 1 73 | (1 row) 74 | 75 | SELECT hll_cardinality(hll_agg(name)) FROM byref; 76 | hll_cardinality 77 | ----------------- 78 | 1 79 | (1 row) 80 | 81 | INSERT INTO byref (uuid, name) VALUES ('fff36e28-db82-43af-ac48-df26a4cda1f3', 'bob'); 82 | INSERT INTO byref (uuid, name) VALUES ('fff36e28-db82-43af-ac48-df26a4cda1f3', 'bob'); 83 | SELECT hll_cardinality(hll_agg(uuid)) FROM byref; 84 | hll_cardinality 85 | ----------------- 86 | 2 87 | (1 row) 88 | 89 | SELECT hll_cardinality(hll_agg(name)) FROM byref; 90 | hll_cardinality 91 | ----------------- 92 | 2 93 | (1 row) 94 | 95 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_min_max.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE stream_cqminmax ( 2 | key text, i8 int8, i4 int4, i2 int2, o oid, f8 float8, f4 float4, 3 | d date, t time, tz timetz, m money, ts timestamp, tstz timestamptz, 4 | ts0 timestamp, ts1 timestamp, txt text, n numeric, a int[]) SERVER pipelinedb; 5 | 6 | CREATE VIEW test_min_max AS SELECT 7 | key::text, 8 | min(i8::int8) AS i8min, max(i8) AS i8max, 9 | min(i4::int4) AS i4min, max(i4) AS i4max, 10 | min(i2::int2) AS i2min, max(i2) AS i2max, 11 | min(o::oid) AS omin, max(o) AS omax, 12 | min(f8::float8) AS f8min, max(f8) AS f8max, 13 | min(f4::float4) AS f4min, max(f4) AS f4max, 14 | min(d::date) AS dmin, max(d) AS dmax, 15 | min(t::time) AS tmin, max(t) AS tmax, 16 | min(tz::timetz) AS tzmin, max(tz) AS tzmax, 17 | min(m::money) AS mmin, max(m) AS mmax, 18 | min(ts::timestamp) AS tsmin, max(ts) AS tsmax, 19 | min(tstz::timestamptz) AS tstzmin, max(tstz) AS tstzmax, 20 | min(ts0::timestamp - ts1::timestamp) AS intervalmin, max(ts0::timestamp - ts1::timestamp) AS intervalmax, 21 | min(txt::text) AS txtmin, max(txt) AS txtmax, 22 | min(n::numeric) AS nmin, max(n) AS nmax, 23 | min(a::int[]) AS amin, max(a) AS amax 24 | FROM stream_cqminmax GROUP BY key; 25 | 26 | INSERT INTO stream_cqminmax 27 | (key, i8, i4, i2, o, f8, f4, d, t, tz, m, ts, tstz, ts1, ts0, txt, n, a) VALUES 28 | ('x', 0, 1, 2, 443, 42.33, 80003.00001, '2014-12-31', '2014-02-28 00:00:00', '2014-02-28 00:00:00-08', 999023.39, '2014-01-01 00:00:00', '2014-01-01 00:00:01-08', '2014-01-01 00:00:00', '2014-01-02 00:00:00', 'first row', -0.00000000042, '{-1, 0, 1}'); 29 | 30 | INSERT INTO stream_cqminmax 31 | (key, i8, i4, i2, o, f8, f4, d, t, tz, m, ts, tstz, ts1, ts0, txt, n, a) VALUES 32 | ('x', 10000, -1, 2, 100000, 442.33, 1e-9, '2014-12-31', '2014-02-28 00:00:00', '2014-02-28 00:00:00-08', 0.389, '2014-01-01 00:00:00', '2014-01-01 00:00:01-08', '2014-01-01 00:00:00', '2014-01-02 00:00:00', 'second row', 0.00000000042, '{-1, 0, 2, 3}'); 33 | 34 | INSERT INTO stream_cqminmax 35 | (key, i8, i4, i2, o, f8, f4, d, t, tz, m, ts, tstz, ts1, ts0, txt, n, a) VALUES 36 | ('y', -1, 1, 2, 443, -0.00001, 1e12, '2010-12-31', '2014-02-28 00:00:00', '2014-02-28 00:00:00-08', -23.40, '2014-01-01 00:00:00', '2014-01-01 00:00:01-08', '2014-01-01 00:00:00', '2014-01-01 00:00:01', 'third row', -0.00000000041, '{-10}'); 37 | 38 | INSERT INTO stream_cqminmax 39 | (key, i8, i4, i2, o, f8, f4, d, t, tz, m, ts, tstz, ts1, ts0, txt, n, a) VALUES 40 | ('y', 0, 1, 2, 443, 442.33, 80003.00001, '2014-12-31', '2014-02-28 00:00:00', '2014-02-28 00:00:00-08', 999023.399, '2014-01-01 00:00:00', '2014-01-01 00:00:01-08', '2014-01-01 00:00:00', '2015-01-02 00:00:00', 'fourth row', 1.00000000042, '{-1, 0, 1}'); 41 | 42 | -- It's hard to read all of the columns of this thing at once, so SELECT a few subsets 43 | SELECT key, i8min, i8max, i4min, i4max, i2min, i2max FROM test_min_max ORDER BY key; 44 | SELECT key, omin, omax, f8min, f8max, f4min, f4max FROM test_min_max ORDER BY key; 45 | SELECT key, dmin, dmax, tmin, tmax, tzmin, tzmax, mmin FROM test_min_max ORDER BY key; 46 | SELECT key, mmax, tsmin, tsmax, tstzmin, tstzmax, intervalmin FROM test_min_max ORDER BY key; 47 | SELECT key, intervalmax, txtmin, txtmax, nmin, nmax, amin, amax FROM test_min_max ORDER BY key; 48 | 49 | DROP FOREIGN TABLE stream_cqminmax CASCADE; 50 | -------------------------------------------------------------------------------- /bin/bootstrap: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import argparse 4 | import os 5 | import shutil 6 | import subprocess 7 | import time 8 | import getpass 9 | import signal 10 | import sys 11 | 12 | 13 | def main(args): 14 | if args.root is None: 15 | p = subprocess.Popen(['pg_config', '--bindir'], stdout=subprocess.PIPE) 16 | out, _ = p.communicate() 17 | if p.returncode == 0: 18 | args.root = os.path.abspath(os.path.join(out.strip(), b'..')) 19 | 20 | root = os.path.expandvars(args.root) 21 | if not os.path.exists(root): 22 | print('Root directory %s does not exist. Did you run `make install`?' % root) 23 | sys.exit(1) 24 | root = os.path.expanduser(root) 25 | print(os.path.join(root, b'data')) 26 | data_dir = os.path.join(root, b'data') 27 | 28 | os.path.exists(data_dir) and shutil.rmtree(data_dir) 29 | try: 30 | subprocess.call(['initdb', '-D', data_dir]) 31 | except OSError: 32 | print('Looks like you forgot to add your PATH? Run the next line and `make bootstrap` again') 33 | print(' export PATH=%s/bin:$PATH' % root) 34 | sys.exit(1) 35 | 36 | proc = subprocess.Popen(['postgres', '-D', data_dir, '-c', 'log_line_prefix=']) 37 | time.sleep(1) 38 | 39 | # This is only required for 'make installcheck'. Before running tests, 40 | # pg_regress wants to do 'DROP DATABASE contrib_regression' when connected to 41 | # the 'postgres' database. With PipelineDB that only works when the background 42 | # workers associated with the 'contrib_regression" database are killed first. 43 | # Otherwise PostgreSQL would either prevent 'DROP DATABASE' or (when FORCEd) 44 | # will result in PipelineDB periodically complaining in the error log about 45 | # background workers not finding a database, which is annoying. Killing the 46 | # corresponding background workers before dropping the database works only if 47 | # the extension is installed in the 'postres' database as well, so the 'DROP 48 | # DATABASE' command is intercepted with a utility hook and 49 | # SignalContQuerySchedulerDropDB() is called for 'contrib_regression'. 50 | subprocess.Popen(['psql', '-c', 51 | 'CREATE EXTENSION IF NOT EXISTS pipelinedb', 'postgres'], 52 | stdout=subprocess.PIPE) 53 | 54 | user = getpass.getuser() 55 | try: 56 | subprocess.check_call(['psql', 'postgres', '-c', 57 | 'CREATE DATABASE %s' % user]) 58 | except subprocess.CalledProcessError: 59 | print('It seems like PostgreSQL failed to start up. Make sure nothing else is running on port 5432, maybe you have an instance of PostgreSQL already running?') 60 | sys.exit(1) 61 | 62 | subprocess.call(['psql', user, '-c', 'CREATE EXTENSION pipelinedb']) 63 | time.sleep(0.1) 64 | 65 | # Cleanup daemons 66 | os.kill(proc.pid, signal.SIGINT) 67 | time.sleep(0.1) 68 | 69 | print("""\nPipelineDB successfully bootstrapped. 70 | Run with `make run`.'""") 71 | 72 | 73 | if __name__ == '__main__': 74 | parser = argparse.ArgumentParser() 75 | parser.add_argument('--root', action='store', dest='root', help='Root directory in which to create the database') 76 | args = parser.parse_args() 77 | main(args) 78 | -------------------------------------------------------------------------------- /src/test/regress/expected/cont_bool_agg.out: -------------------------------------------------------------------------------- 1 | -- bit_and, bit_or 2 | CREATE FOREIGN TABLE bit_stream_cqboolagg (k text, b bit) SERVER pipelinedb; 3 | CREATE VIEW test_bit_and AS SELECT k::text, bit_and(b::bit) FROM bit_stream_cqboolagg GROUP BY k; 4 | CREATE VIEW test_bit_or AS SELECT k::text, bit_or(b::bit) FROM bit_stream_cqboolagg GROUP BY k; 5 | INSERT INTO bit_stream_cqboolagg (k, b) VALUES ('x', 1::bit), ('x', '1'::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit); 6 | INSERT INTO bit_stream_cqboolagg (k, b) VALUES ('y', 0::bit), ('y', 1::bit), ('y', 0::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit), ('x', 1::bit); 7 | SELECT * FROM test_bit_and ORDER BY k DESC; 8 | k | bit_and 9 | ---+--------- 10 | y | 0 11 | x | 1 12 | (2 rows) 13 | 14 | SELECT * FROM test_bit_or ORDER BY k DESC; 15 | k | bit_or 16 | ---+-------- 17 | y | 1 18 | x | 1 19 | (2 rows) 20 | 21 | INSERT INTO bit_stream_cqboolagg (k, b) VALUES ('x', 1::bit), ('x', 0::bit); 22 | INSERT INTO bit_stream_cqboolagg (k, b) VALUES ('y', 0::bit); 23 | SELECT * FROM test_bit_and ORDER BY k DESC; 24 | k | bit_and 25 | ---+--------- 26 | y | 0 27 | x | 0 28 | (2 rows) 29 | 30 | SELECT * FROM test_bit_or ORDER BY k DESC; 31 | k | bit_or 32 | ---+-------- 33 | y | 1 34 | x | 1 35 | (2 rows) 36 | 37 | DROP FOREIGN TABLE bit_stream_cqboolagg CASCADE; 38 | NOTICE: drop cascades to 2 other objects 39 | DETAIL: drop cascades to view test_bit_and 40 | drop cascades to view test_bit_or 41 | -- bool_and, bool_or, every 42 | CREATE FOREIGN TABLE bool_stream_cqboolagg (k text, b boolean) SERVER pipelinedb; 43 | CREATE VIEW test_bool_and AS SELECT k::text, bool_and(b::boolean) FROM bool_stream_cqboolagg GROUP BY k; 44 | CREATE VIEW test_bool_or AS SELECT k::text, bool_or(b::boolean) FROM bool_stream_cqboolagg GROUP BY k; 45 | CREATE VIEW test_every AS SELECT k::text, every(b::boolean) FROM bool_stream_cqboolagg GROUP BY k; 46 | INSERT INTO bool_stream_cqboolagg (k, b) VALUES ('x', 't'), ('x', 't'), ('x', true), ('x', 't'), ('x', 't'), ('x', true), ('x', true); 47 | INSERT INTO bool_stream_cqboolagg (k, b) VALUES ('y', false), ('y', 'f'), ('y', 'f'), ('x', false), ('x', true), ('x', 't'), ('x', 't'); 48 | SELECT * FROM test_bool_and ORDER BY k DESC; 49 | k | bool_and 50 | ---+---------- 51 | y | f 52 | x | f 53 | (2 rows) 54 | 55 | SELECT * FROM test_bool_or ORDER BY k DESC; 56 | k | bool_or 57 | ---+--------- 58 | y | f 59 | x | t 60 | (2 rows) 61 | 62 | SELECT * FROM test_every ORDER BY k DESC; 63 | k | every 64 | ---+------- 65 | y | f 66 | x | f 67 | (2 rows) 68 | 69 | INSERT INTO bool_stream_cqboolagg (k, b) VALUES ('x', 't'), ('x', 'f'); 70 | INSERT INTO bool_stream_cqboolagg (k, b) VALUES ('y', 'f'); 71 | SELECT * FROM test_bool_and ORDER BY k DESC; 72 | k | bool_and 73 | ---+---------- 74 | y | f 75 | x | f 76 | (2 rows) 77 | 78 | SELECT * FROM test_bool_or ORDER BY k DESC; 79 | k | bool_or 80 | ---+--------- 81 | y | f 82 | x | t 83 | (2 rows) 84 | 85 | SELECT * FROM test_every ORDER BY k DESC; 86 | k | every 87 | ---+------- 88 | y | f 89 | x | f 90 | (2 rows) 91 | 92 | DROP FOREIGN TABLE bool_stream_cqboolagg CASCADE; 93 | NOTICE: drop cascades to 3 other objects 94 | DETAIL: drop cascades to view test_bool_and 95 | drop cascades to view test_bool_or 96 | drop cascades to view test_every 97 | -------------------------------------------------------------------------------- /src/test/regress/expected/sw_expiration.out: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE sw_vacuum_stream (key text) SERVER pipelinedb; 2 | CREATE VIEW sw_vacuum AS SELECT key, COUNT(*) FROM sw_vacuum_stream WHERE arrival_timestamp > clock_timestamp() - interval '3 second' GROUP BY key; 3 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 4 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 5 | SELECT pg_sleep(1); 6 | pg_sleep 7 | ---------- 8 | 9 | (1 row) 10 | 11 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 12 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 13 | SELECT * FROM sw_vacuum ORDER BY key; 14 | key | count 15 | -----+------- 16 | a | 4 17 | b | 4 18 | c | 4 19 | (3 rows) 20 | 21 | -- Just verify that the mrel has more rows, as we can't gaurantee which time bucket the 22 | -- rows will fall in which makes it tricky to compare this result to a predetermined result 23 | SELECT (SELECT COUNT(*) FROM sw_vacuum) < (SELECT COUNT(*) FROM sw_vacuum_mrel); 24 | ?column? 25 | ---------- 26 | t 27 | (1 row) 28 | 29 | SELECT DISTINCT key FROM sw_vacuum_mrel ORDER BY key; 30 | key 31 | ----- 32 | a 33 | b 34 | c 35 | (3 rows) 36 | 37 | SELECT pg_sleep(3); 38 | pg_sleep 39 | ---------- 40 | 41 | (1 row) 42 | 43 | SELECT 0 * pipelinedb.ttl_expire('sw_vacuum'); 44 | ?column? 45 | ---------- 46 | 0 47 | (1 row) 48 | 49 | SELECT * FROM sw_vacuum ORDER BY key; 50 | key | count 51 | -----+------- 52 | (0 rows) 53 | 54 | SELECT key, SUM(count) FROM sw_vacuum_mrel GROUP BY key ORDER BY key; 55 | key | sum 56 | -----+----- 57 | (0 rows) 58 | 59 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 60 | SELECT pg_sleep(1); 61 | pg_sleep 62 | ---------- 63 | 64 | (1 row) 65 | 66 | INSERT INTO sw_vacuum_stream (key) VALUES ('a'), ('b'), ('c'); 67 | SELECT * FROM sw_vacuum ORDER BY key; 68 | key | count 69 | -----+------- 70 | a | 2 71 | b | 2 72 | c | 2 73 | (3 rows) 74 | 75 | SELECT (SELECT COUNT(*) FROM sw_vacuum) < (SELECT COUNT(*) FROM sw_vacuum_mrel); 76 | ?column? 77 | ---------- 78 | t 79 | (1 row) 80 | 81 | SELECT DISTINCT key FROM sw_vacuum_mrel ORDER BY key; 82 | key 83 | ----- 84 | a 85 | b 86 | c 87 | (3 rows) 88 | 89 | SELECT 0 * pipelinedb.ttl_expire('sw_vacuum'); 90 | ?column? 91 | ---------- 92 | 0 93 | (1 row) 94 | 95 | SELECT * FROM sw_vacuum ORDER BY key; 96 | key | count 97 | -----+------- 98 | a | 2 99 | b | 2 100 | c | 2 101 | (3 rows) 102 | 103 | SELECT key, SUM(count) FROM sw_vacuum_mrel GROUP BY key ORDER BY key; 104 | key | sum 105 | -----+----- 106 | a | 2 107 | b | 2 108 | c | 2 109 | (3 rows) 110 | 111 | SELECT pg_sleep(3); 112 | pg_sleep 113 | ---------- 114 | 115 | (1 row) 116 | 117 | SELECT 0 * pipelinedb.ttl_expire('sw_vacuum'); 118 | ?column? 119 | ---------- 120 | 0 121 | (1 row) 122 | 123 | SELECT * FROM sw_vacuum ORDER BY key; 124 | key | count 125 | -----+------- 126 | (0 rows) 127 | 128 | SELECT key, SUM(count) FROM sw_vacuum_mrel GROUP BY key ORDER BY key; 129 | key | sum 130 | -----+----- 131 | (0 rows) 132 | 133 | DROP FOREIGN TABLE sw_vacuum_stream CASCADE; 134 | NOTICE: drop cascades to view sw_vacuum 135 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_complex_types.sql: -------------------------------------------------------------------------------- 1 | -- Test complex types 2 | CREATE TYPE test_cont_complex_type AS ( 3 | x int, 4 | y int, 5 | z text 6 | ); 7 | 8 | CREATE FOREIGN TABLE cont_complex_stream (r test_cont_complex_type, x int, y int, z text) SERVER pipelinedb; 9 | CREATE VIEW test_cont_complex1 AS SELECT bloom_agg(r::test_cont_complex_type) FROM cont_complex_stream; 10 | 11 | INSERT INTO cont_complex_stream (r) VALUES ((1, 1, 'hello')); 12 | INSERT INTO cont_complex_stream (r) VALUES ((1, 2, 'world')::test_cont_complex_type); 13 | 14 | SELECT bloom_cardinality(bloom_agg) FROM test_cont_complex1; 15 | 16 | -- These shouldn't change the cardinality 17 | INSERT INTO cont_complex_stream (r) VALUES ((1, 1, 'hello')); 18 | INSERT INTO cont_complex_stream (r) VALUES ((1, 2, 'world')); 19 | INSERT INTO cont_complex_stream (r) VALUES ((1, 1, 'hello')::test_cont_complex_type); 20 | INSERT INTO cont_complex_stream (r) VALUES ((1, 2, 'world')::test_cont_complex_type); 21 | 22 | SELECT bloom_cardinality(bloom_agg) FROM test_cont_complex1; 23 | 24 | -- Null check 25 | INSERT INTO cont_complex_stream (r) VALUES (NULL); 26 | 27 | SELECT bloom_cardinality(bloom_agg) FROM test_cont_complex1; 28 | 29 | -- These should be true 30 | SELECT bloom_contains(bloom_agg, (1, 1, 'hello'::text)) FROM test_cont_complex1; 31 | SELECT bloom_contains(bloom_agg, (1, 2, 'world'::text)) FROM test_cont_complex1; 32 | SELECT bloom_contains(bloom_agg, (1, 1, 'hello')::test_cont_complex_type) FROM test_cont_complex1; 33 | SELECT bloom_contains(bloom_agg, (1, 2, 'world')::test_cont_complex_type) FROM test_cont_complex1; 34 | -- These should be false 35 | -- This is because the type for 'hello' will be 'unknown' and so it doesn't 36 | -- have the same hash value as the equivalent constant of type 'text' 37 | SELECT bloom_contains(bloom_agg, (1, 1, 'hello')) FROM test_cont_complex1; 38 | SELECT bloom_contains(bloom_agg, (1, 2, 'world')) FROM test_cont_complex1; 39 | 40 | -- Test anonymous RECORD types 41 | CREATE VIEW test_cont_complex2 AS SELECT bloom_agg((x::int, y::int)) FROM cont_complex_stream; 42 | CREATE VIEW test_cont_complex3 AS SELECT bloom_agg((x::int, y::int, z::text)) FROM cont_complex_stream; 43 | 44 | INSERT INTO cont_complex_stream (x, y, z) VALUES (1, 1, 'hello'); 45 | INSERT INTO cont_complex_stream (x, y, z) VALUES (1, 2, 'world'); 46 | 47 | SELECT bloom_cardinality(bloom_agg) FROM test_cont_complex2; 48 | SELECT bloom_cardinality(bloom_agg) FROM test_cont_complex3; 49 | 50 | -- All should be true 51 | SELECT bloom_contains(bloom_agg, (1, 1)) FROM test_cont_complex2; 52 | SELECT bloom_contains(bloom_agg, (1, 2)) FROM test_cont_complex2; 53 | SELECT bloom_contains(bloom_agg, (1, 1, 'hello'::text)) FROM test_cont_complex3; 54 | SELECT bloom_contains(bloom_agg, (1, 2, 'world'::text)) FROM test_cont_complex3; 55 | 56 | -- Test non-pass by value + non-complex type 57 | CREATE VIEW test_cont_complex4 AS SELECT bloom_agg(z::text) FROM cont_complex_stream; 58 | 59 | INSERT INTO cont_complex_stream (z) VALUES ('hello'), ('world'); 60 | 61 | SELECT bloom_cardinality(bloom_agg) FROM test_cont_complex4; 62 | 63 | SELECT bloom_contains(bloom_agg, 'hello'::text) FROM test_cont_complex4; 64 | SELECT bloom_contains(bloom_agg, 'world'::text) FROM test_cont_complex4; 65 | 66 | DROP FOREIGN TABLE cont_complex_stream CASCADE; 67 | DROP TYPE test_cont_complex_type; 68 | -------------------------------------------------------------------------------- /src/test/py/test_cont_transform.py: -------------------------------------------------------------------------------- 1 | from base import pipeline, clean_db 2 | import os 3 | import tempfile 4 | import time 5 | 6 | 7 | def test_multiple_insert(pipeline, clean_db): 8 | pipeline.create_stream('stream0', x='int') 9 | pipeline.create_stream('stream1', x='int') 10 | pipeline.create_stream('stream2', x='int') 11 | 12 | pipeline.create_cv('cv0', 'SELECT count(*) FROM stream1') 13 | pipeline.create_cv('cv1', 'SELECT count(*) FROM stream2') 14 | pipeline.create_ct('ct1', 'SELECT x::int FROM stream0 WHERE mod(x, 2) = 0', "pipelinedb.insert_into_stream('stream1', 'stream2')") 15 | 16 | pipeline.insert('stream0', ('x',), [(n,) for n in range(1000)]) 17 | 18 | count = pipeline.execute('SELECT count FROM cv0')[0]['count'] 19 | assert count == 500 20 | count = pipeline.execute('SELECT count FROM cv1')[0]['count'] 21 | assert count == 500 22 | 23 | 24 | def test_nested_transforms(pipeline, clean_db): 25 | pipeline.create_stream('stream0', x='int') 26 | pipeline.create_stream('stream2', x='int') 27 | pipeline.create_stream('stream4', x='int') 28 | 29 | pipeline.create_cv('cv0', 'SELECT count(*) FROM stream4') 30 | pipeline.create_cv('cv1', 'SELECT count(*) FROM stream2') 31 | pipeline.create_ct('ct0', 'SELECT x::int FROM stream2 WHERE mod(x, 4) = 0', 32 | "pipelinedb.insert_into_stream('stream4')") 33 | pipeline.create_ct('ct1', 'SELECT x::int FROM stream0 WHERE mod(x, 2) = 0', 34 | "pipelinedb.insert_into_stream('stream2')") 35 | 36 | pipeline.insert('stream0', ('x',), [(n,) for n in range(1000)]) 37 | 38 | count = pipeline.execute('SELECT count FROM cv0')[0]['count'] 39 | assert count == 250 40 | count = pipeline.execute('SELECT count FROM cv1')[0]['count'] 41 | assert count == 500 42 | 43 | 44 | def test_deadlock_regress(pipeline, clean_db): 45 | nitems = 2000000 46 | tmp_file = os.path.join(tempfile.gettempdir(), 'tmp.json') 47 | query = 'SELECT generate_series(1, %d) AS n' % nitems 48 | pipeline.execute("COPY (%s) TO '%s'" % (query, tmp_file)) 49 | 50 | pipeline.create_stream('s1', n='int') 51 | pipeline.create_stream('s2', n='int') 52 | pipeline.create_ct('ct', 'SELECT n FROM s1 WHERE n IS NOT NULL', 53 | "pipelinedb.insert_into_stream('s2')") 54 | pipeline.create_cv('cv', 'SELECT count(*) FROM s2') 55 | 56 | for copy in [True, False]: 57 | for nworkers in [1, 4]: 58 | for sync in ['receive', 'commit']: 59 | pipeline.stop() 60 | pipeline.run({ 61 | 'pipelinedb.num_workers': nworkers, 62 | 'pipelinedb.stream_insert_level': 'sync_%s' % sync 63 | }) 64 | 65 | pipeline.execute("SELECT pipelinedb.truncate_continuous_view('cv')") 66 | pipeline.execute('COMMIT') 67 | 68 | if copy: 69 | pipeline.execute("COPY s1 (n) FROM '%s'" % tmp_file) 70 | else: 71 | pipeline.execute('INSERT INTO s1 (n) %s' % query) 72 | 73 | count = dict(pipeline.execute('SELECT count FROM cv')[0] or {}) 74 | ntries = 5 75 | while count.get('count') != nitems and ntries > 0: 76 | assert sync == 'receive' 77 | time.sleep(1) 78 | count = dict(pipeline.execute('SELECT count FROM cv')[0] or {}) 79 | ntries -= 1 80 | assert count and count['count'] == nitems 81 | 82 | os.remove(tmp_file) 83 | 84 | pipeline.stop() 85 | pipeline.run() 86 | -------------------------------------------------------------------------------- /src/test/regress/sql/cont_sw_min_max.sql: -------------------------------------------------------------------------------- 1 | CREATE FOREIGN TABLE sw_minmax_stream ( 2 | key text, i8 int8, i4 int4, i2 int2, o oid, f8 float8, f4 float4, 3 | d date, t time, tz timetz, m money, ts timestamp, tstz timestamptz, 4 | ts0 timestamp, ts1 timestamp, txt text, n numeric, a int[]) SERVER pipelinedb; 5 | 6 | CREATE VIEW test_sw_min_max AS SELECT 7 | key::text, 8 | min(i8::int8) AS i8min, max(i8) AS i8max, 9 | min(i4::int4) AS i4min, max(i4) AS i4max, 10 | min(i2::int2) AS i2min, max(i2) AS i2max, 11 | min(o::oid) AS omin, max(o) AS omax, 12 | min(f8::float8) AS f8min, max(f8) AS f8max, 13 | min(f4::float4) AS f4min, max(f4) AS f4max, 14 | min(d::date) AS dmin, max(d) AS dmax, 15 | min(t::time) AS tmin, max(t) AS tmax, 16 | min(tz::timetz) AS tzmin, max(tz) AS tzmax, 17 | min(m::money) AS mmin, max(m) AS mmax, 18 | min(ts::timestamp) AS tsmin, max(ts) AS tsmax, 19 | min(tstz::timestamptz) AS tstzmin, max(tstz) AS tstzmax, 20 | min(ts0::timestamp - ts1::timestamp) AS intervalmin, max(ts0::timestamp - ts1::timestamp) AS intervalmax, 21 | min(txt::text) AS txtmin, max(txt) AS txtmax, 22 | min(n::numeric) AS nmin, max(n) AS nmax, 23 | min(a::int[]) AS amin, max(a) AS amax 24 | FROM sw_minmax_stream WHERE arrival_timestamp > clock_timestamp() - interval '60 second' GROUP BY key; 25 | 26 | INSERT INTO sw_minmax_stream 27 | (key, i8, i4, i2, o, f8, f4, d, t, tz, m, ts, tstz, ts1, ts0, txt, n, a) VALUES 28 | ('x', 0, 1, 2, 443, 42.33, 80003.00001, '2014-12-31', '2014-02-28 00:00:00', '2014-02-28 00:00:00-08', 999023.39, '2014-01-01 00:00:00', '2014-01-01 00:00:01-08', '2014-01-01 00:00:00', '2014-01-02 00:00:00', 'first row', -0.00000000042, '{-1, 0, 1}'); 29 | 30 | INSERT INTO sw_minmax_stream 31 | (key, i8, i4, i2, o, f8, f4, d, t, tz, m, ts, tstz, ts1, ts0, txt, n, a) VALUES 32 | ('x', 10000, -1, 2, 100000, 442.33, 1e-9, '2014-12-31', '2014-02-28 00:00:00', '2014-02-28 00:00:00-08', 0.389, '2014-01-01 00:00:00', '2014-01-01 00:00:01-08', '2014-01-01 00:00:00', '2014-01-02 00:00:00', 'second row', 0.00000000042, '{-1, 0, 2, 3}'); 33 | 34 | SELECT pg_sleep(1); 35 | 36 | INSERT INTO sw_minmax_stream 37 | (key, i8, i4, i2, o, f8, f4, d, t, tz, m, ts, tstz, ts1, ts0, txt, n, a) VALUES 38 | ('y', -1, 1, 2, 443, -0.00001, 1e12, '2010-12-31', '2014-02-28 00:00:00', '2014-02-28 00:00:00-08', -23.40, '2014-01-01 00:00:00', '2014-01-01 00:00:01-08', '2014-01-01 00:00:00', '2014-01-01 00:00:01', 'third row', -0.00000000041, '{-10}'); 39 | 40 | INSERT INTO sw_minmax_stream 41 | (key, i8, i4, i2, o, f8, f4, d, t, tz, m, ts, tstz, ts1, ts0, txt, n, a) VALUES 42 | ('y', 0, 1, 2, 443, 442.33, 80003.00001, '2014-12-31', '2014-02-28 00:00:00', '2014-02-28 00:00:00-08', 999023.399, '2014-01-01 00:00:00', '2014-01-01 00:00:01-08', '2014-01-01 00:00:00', '2015-01-02 00:00:00', 'fourth row', 1.00000000042, '{-1, 0, 1}'); 43 | 44 | -- It's hard to read all of the columns of this thing at once, so SELECT a few subsets 45 | SELECT key, i8min, i8max, i4min, i4max, i2min, i2max FROM test_sw_min_max ORDER BY key; 46 | SELECT key, omin, omax, f8min, f8max, f4min, f4max FROM test_sw_min_max ORDER BY key; 47 | SELECT key, dmin, dmax, tmin, tmax, tzmin, tzmax, mmin FROM test_sw_min_max ORDER BY key; 48 | SELECT key, mmax, tsmin, tsmax, tstzmin, tstzmax, intervalmin FROM test_sw_min_max ORDER BY key; 49 | SELECT key, intervalmax, txtmin, txtmax, nmin, nmax, amin, amax FROM test_sw_min_max ORDER BY key; 50 | 51 | DROP FOREIGN TABLE sw_minmax_stream CASCADE; 52 | --------------------------------------------------------------------------------