├── .gitignore ├── CHANGES ├── INSTALL ├── LICENSE ├── META.json ├── Makefile ├── README.md ├── about.txt ├── btree.c ├── btree.h ├── disk.c ├── disk.h ├── example.sql ├── expected ├── create.out ├── cumagg.out ├── datetime.out ├── drop.out ├── grandagg.out ├── gridagg.out ├── groupbyagg.out ├── hashagg.out ├── math.out ├── operators.out ├── scalarop.out ├── sort.out ├── span.out ├── spec.out ├── transform.out └── windowagg.out ├── fileio.c ├── fileio.h ├── func.c ├── func.h ├── imcs--1.1.sql ├── imcs.c ├── imcs.conf ├── imcs.control ├── imcs.h ├── install.sh ├── smp.c ├── smp.h ├── sql ├── create.sql ├── cumagg.sql ├── datetime.sql ├── drop.sql ├── grandagg.sql ├── gridagg.sql ├── groupbyagg.sql ├── hashagg.sql ├── math.sql ├── operators.sql ├── scalarop.sql ├── sort.sql ├── span.sql ├── spec.sql ├── transform.sql └── windowagg.sql ├── sysv_shmem.patch ├── threadpool.c ├── tpch.sql └── user_guide.html /.gitignore: -------------------------------------------------------------------------------- 1 | # Object files 2 | *.o 3 | *.ko 4 | 5 | # Libraries 6 | *.lib 7 | *.a 8 | 9 | # Shared objects (inc. Windows DLLs) 10 | *.dll 11 | *.so 12 | *.so.* 13 | *.dylib 14 | 15 | # Executables 16 | *.exe 17 | *.out 18 | *.app 19 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | --- Release version 1.01 (09.12.2013) ------------------------------------- 2 | 3 | Initial version 4 | 5 | --- Release version 1.02 (29.12.2013) ------------------------------------- 6 | 7 | 1. Add support of MONEY type 8 | 2. Add support of VARCHAR type 9 | 3. Implement join by timestamp 10 | 4. Correctly handle NULL in TABLE_get(id) function 11 | 5. Correctly handle materialized views 12 | 6. Fix bug in autoload 13 | 7. Add EMA and ATR indicators 14 | 8. Support non-character timeseries ID 15 | 9. Fix bug in infinite timeseries detection 16 | 10. Add more overloaded binary operators accepting numeric parameter 17 | 11. Support grand, group and hash ALL/ANY aggregates. 18 | 12. Make timestamp join more flexible: add direction parameter 19 | 13. Reimplement insert trigger in C 20 | 21 | --- Release version 1.03 (31.01.2014) ------------------------------------- 22 | 1. Take in account extra_float_digits for printing floating point values 23 | 2. Add cs_filter_first_pos function 24 | 3. Support disk mode 25 | 4. Support RLE compression 26 | 27 | --- Release version 1.04 (05.03.2014) ------------------------------------- 28 | 1. Add cs_trend function 29 | 2. Fix hang in case of calculating aggregates for unbounded sequence 30 | 3. Fix incorrect construction of string constants 31 | 4. Support bool type 32 | 5. Fix bug in parallel executor causing crash in case of empty result 33 | 6. Add TABLE_is_loaded() function 34 | 7. Fix IMCS context cleanup bug in case of nested SPI calls 35 | 8. Add limit parameter to XXX_get() function 36 | 37 | --- Release version 1.05 (19.04.2014) ------------------------------------- 38 | 1. Fix bugs in cs_hash_dup_count 39 | 2. Fix handling of scalar types in cs_project 40 | 3. Support cast to character type 41 | 4. Thread safe error reporting 42 | 5. Add TABLE_id() and TABLE_timestamp() functions 43 | 6. Support load of unlimited varying size strings to columnar store using dictionary 44 | 7. Fix synchronization issue in delete function 45 | 8. Fix bugs in RLE branch of imcs_delete_page and lack of range check in imcs_delete 46 | 9. Fix bug in imcs_search 47 | 48 | --- Release version 1.06 (10.07.2014) ------------------------------------- 49 | 1. Fix problem with "imcs.use_rle" property. 50 | 2. Implicitly convert code to string for LIKE operator. 51 | 3. Add cs_dictionary_size() function. 52 | 4. Remove overloaded version of || operator with scalars 53 | 5. Fix mistypings in documentation 54 | 55 | 56 | --- Release version 1.07 (??.07.2014) ------------------------------------- 57 | 1. Change semantic of comparison operations for character types: treat string literal as constant rather than text representation of sequence 58 | 2. Add cs_as_array function 59 | 3. Make it possible to add columns to columnar store without reloading all data 60 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Install PostgreSQL from source code in PGSQLSRC and execute following commands: 2 | 3 | % cd PGSQLSRC/contrib 4 | % tar xzvf imcs.tgz 5 | % cd imcs 6 | % ./install.sh 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /META.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "imcs", 3 | "abstract": "In-Memory Columnar Store", 4 | "description": "IMCS provides columnar (vertical) store for PostgreSQL. It allows to reach 10-100 advantages in performance because of avoiding disk and MVCC overhead, vector operations, parallel execution and data skipping. IMCS providers a wide range of analytic operators.", 5 | "version": "0.1.7", 6 | "maintainer": ["Konstantin Kninzhik "], 7 | "license": { 8 | "PostgreSQL": "http://www.postgresql.org/about/licence" 9 | }, 10 | "release_status": "stable", 11 | "provides": { 12 | "imcs": { 13 | "file": "imcs--1.1.sql", 14 | "docfile": "user_guide.html", 15 | "version": "0.1.7", 16 | "abstract": "In-Memory Columnar Store" 17 | } 18 | }, 19 | "tags": ["columnar store", "vertical", "in-memory", "analytic", "OLAP", "IMCS", "timeseries"], 20 | "resources": { 21 | "repository": { 22 | "url": "https://github.com/knizhnik/imcs.git", 23 | "web": "https://github.com/knizhnik/imcs", 24 | "type": "git" 25 | }, 26 | "bugtracker": { 27 | "web": "https://github.com/knizhnik/imcs/issues" 28 | } 29 | }, 30 | "generated_by": "Konstantin Knizhnik", 31 | "meta-spec": { 32 | "version": "1.0.0", 33 | "url": "http://pgxn.org/meta/spec.txt" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # *------------------------------------------------------------------------- 2 | # * 3 | # * Makefile 4 | # * 5 | # * 6 | # * 7 | # * Copyright (c) 2013, Konstantin Knizhnik, ww.garret.ru 8 | # * Author: Konstantin Knizhnik 9 | # * 10 | # * $Id: Makefile 28 2013-10-10 17:18:31Z lptolik $ 11 | # * 12 | # *------------------------------------------------------------------------- 13 | 14 | MODULE_big = imcs 15 | 16 | PG_CPPFLAGS += -O0 -Wall -pthread 17 | IMCS_VERSION=1.06 18 | 19 | ifdef USE_DISK 20 | OBJS = imcs.o func.o smp.o btree.o threadpool.o fileio.o disk.o 21 | PG_CPPFLAGS += -DIMCS_DISK_SUPPORT 22 | else 23 | OBJS = imcs.o func.o smp.o btree.o threadpool.o 24 | endif 25 | 26 | EXTENSION = imcs 27 | DATA = imcs--1.1.sql 28 | REGRESS = create span operators math datetime transform scalarop grandagg groupbyagg gridagg windowagg hashagg cumagg sort spec drop 29 | REGRESS_OPTS = --temp-config $(top_srcdir)/contrib/imcs/imcs.conf 30 | 31 | SHLIB_LINK += $(filter -lm, $(LIBS)) 32 | 33 | ifdef USE_PGXS 34 | PG_CONFIG = pg_config 35 | PGXS := $(shell $(PG_CONFIG) --pgxs) 36 | include $(PGXS) 37 | else 38 | subdir = contrib/imcs 39 | top_builddir = ../.. 40 | include $(top_builddir)/src/Makefile.global 41 | include $(top_srcdir)/contrib/contrib-global.mk 42 | endif 43 | 44 | distrib: 45 | rm -f *.o 46 | rm -rf results/ regression.diffs regression.out tmp_check/ log/ 47 | cd .. ; tar --exclude=.svn -chvzf imcs-$(IMCS_VERSION).tar.gz imcs 48 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | imcs 2 | ==== 3 | 4 | In-Memory Columnar Store extension for PostgreSQL 5 | -------------------------------------------------------------------------------- /about.txt: -------------------------------------------------------------------------------- 1 | In-Memory Columnar Store for PostgreSQL 2 | 3 | Columnar store manager stores data tables as sections of columns of data rather than as rows of data. 4 | Most of traditional DBMS-es store data in rows ("horizontally"): all record attributes are stored together. 5 | Such approach allows to load the whole record using one read operation which usually leads to better performance for OLTP 6 | queries (which access or update single records). But OLAP queries are mostly performing operations on individual columns, 7 | for example calculating sum or average of some column. In this case vertical data representation, when data for each column 8 | is stored independently, is more efficient. There are several DBMS-es in marker which are based on vertical model: Vertica, 9 | SciDB,... Also most of mainstream commercial databases also provide OLAP extensions based on vertical storage: 10 | Blue Acceleration for DB2, Oracle Database In-Memory Option, Microsoft SQL server column store... 11 | 12 | Columnar store or vertical representation of data allows to achieve better performance in comparison with classical horizontal representation due to three factors: 13 | * Reducing size of fetched data: only columns involved in query are accessed. 14 | * Vector operations. Applying an operator to set of values (tile) makes it possible to minimize interpretation cost. 15 | Also SIMD instructions of modern processors accelerate execution of vector operations. 16 | * Compression of data. Certainly compression can also be used for all the records, but independent compression of each column can give much better results without significant extra CPU overhead. For example such simple compression algorithm like RLE 17 | (run-length-encoding) allows not only to reduce used space, but also minimize number of performed operations. 18 | 19 | Modern servers usually have large amount of operating memory. Server with terabyte of RAM is not something very exotic now, 20 | especially in financial world. So volumes of RAM which few years ago are available only at supercomputers can now met in configurations 21 | of standard servers... Certainly there are databases which requires petabyte and more memory, which still can not fit in RAM. 22 | But size of most databases is comparable with size of available RAM, which allows to keep all data in memory. 23 | It requires special algorithms for main-memory databases. For example it is possible to configure size of buffers (disk cache) 24 | in traditional database to be large enough to fit the complete database in memory. But still algorithms oriented on work with disk 25 | and optimized to reduce number of IO operations will be inefficient for in-memory data and have to do a lot of redundant work. 26 | 27 | IMCS provides columnar store for PostgreSQL. 28 | There is some specific of PostgreSQL which affects design of IMCS: 29 | 30 | * PostgreSQL is not able to parallelize execution of an individual query. It may be normal for OLTP where larger number of simples 31 | queries are usually executed concurrently. For for OLAP situation is quite opposite: there is small number of queries but execution of 32 | each query may take significant amount of time. 33 | * PostgreSQL has multiversion concurrency control (MVCC). It allows to execute larger number of queries concurrently, 34 | but adds significant space and CPU overhead. As it was mentioned above, in case of OLAP we rarely need concurrent execution of multiple queries, but it is important to make execution of single complex query as fast as possible and so minimize locking overhead. 35 | * MVCC nature of PostgreSQL makes it difficult to work with large updatable objects (because MVCC requires cloning of an object 36 | for each update). So it is not possible to efficiently work with columnar store data using standard PostgreSQL data model. 37 | 38 | Taken in account all above we decided to 39 | * Store MVCC data in PostgreSQL shared memory (to allow access to it from all PostgreSQL server processes). 40 | * Provide own executor for columnar store functions allowing to use vector operations and execute them concurrently by multiple thread. 41 | * Do not touch standard PostgreSQL query executor and optimizer. 42 | 43 | So IMCS is implemented as standard PostgreSQL extension, providing set of special functions and operators for columnar data. 44 | Some of these functions and operators are analog of standard SQL operators. For example binary and unary arithmetic operators, 45 | comparisons, match string operations, sorting, aggregation, inserting/deleting data. 46 | But there are a number of sophisticated analytic operators, for example finding extremum, crosses with zero, building histogram, 47 | calculating ranks, percentiles, ... IMCS provides extended set of aggregates which are required for financial application, 48 | like calculation of split-adjusted price, volume-weighted average price, moving average... All this aggregates can be calculated in parallel, utilizing all available CPU cores. 49 | 50 | IMCS is first of all oriented on work with timeseries. Timeseries is sequence of usually small fixed size elements ordered by some 51 | timestamp. Operations with timeseries rarely access some particular timeseries element, instead of it them operate either with whole 52 | timeseries either with some time interval. Such specific of timeseries operation requires special index for timeseries, 53 | which is different from traditional database indexes. Such index should not provide efficient way of locating arbitrary timeseries 54 | element. Instead of it this index should be able to efficiently extract range of timeseries elements. 55 | 56 | Our experiments show that IMCS provides 10-100 times improvement in performance comparing with standard SQL queries. 57 | The larger size of manipulated data is, the large is advantage in performance. If for timeseries with thousands elements improvement 58 | is about ten times, for timeseries with millions elements advantage in performance is more than 100 times. 59 | 60 | IMCS is integrated with standard SQL model, providing functions to switch from horizontal to vertical representation and vice versa. 61 | So IMCS usage pattern is the following: 62 | 63 | * You should first of all choose which tables in your database requires vertical representation. If you wan to have vertical representation only for subset of table columns, then create a view. 64 | * IMCS generates PL/pgSQL functions to work with vertical representation of this table. 65 | * Data is loaded in PostgreSQL table in standard way (imported from CSV file, inserted using SQL,...). 66 | * Either automatically using trigger, either explicitly calling IMCS load function, this data is inserted in columnar store. 67 | * You query this data using IMCS functions and operators. Result of such query can be a tuple with timeseries. Or alternatively it is 68 | possible to flip this result back to horizontal representation, so you will get standard SQL set of tuples. In the last case it is possible 69 | to continue processing of results using standard SQL queries. Also you can convert timeseries to PostgreSQL array which may be useful 70 | to deal with timeseries from programming languages which API already supports arrays. 71 | 72 | Advantages and disadvantages of IMCS approach: 73 | 74 | + Fast execution based on vector operations 75 | + Parallel execution of query (for some operations) 76 | + No changes in PostgreSQL core (just standard extension) 77 | + No MVCC overhead (MURSIW isolation level) 78 | + No disk IO (in-memory store) 79 | + Optimized for timeseries (massive operations with time slices) 80 | 81 | - Requires special queries (standard SQL queries can not be executed for IMCS data) 82 | - Need to reload data on server restart 83 | - Size of database is limited by size of RAM 84 | - Maintaining alternative representation of data requires extra memory and CPU. 85 | - Less level of concurrency for execution of multiple queries (it is not possible for example to append and query some table at the same time) 86 | - Not able to perform joins, index searches (only by timestamp) 87 | - Supports only fixed size string types 88 | -------------------------------------------------------------------------------- /btree.c: -------------------------------------------------------------------------------- 1 | #include "btree.h" 2 | #include "disk.h" 3 | #include 4 | 5 | static bool imcs_next_tile(imcs_iterator_h iterator) 6 | { 7 | int i; 8 | imcs_iterator_context_t* ctx; 9 | if (iterator->first_pos > iterator->last_pos) { 10 | return false; 11 | } 12 | ctx = (imcs_iterator_context_t*)iterator->context; 13 | i = ctx->stack_size-1; 14 | if (i >= 0 && iterator->next_pos <= iterator->last_pos) { 15 | size_t tile_size = 0, n_vals; 16 | while (true) { 17 | imcs_page_t* pg; 18 | size_t inc = 0; 19 | while (true) { 20 | pg = ctx->stack[i].page; 21 | IMCS_LOAD_PAGE(pg); 22 | ctx->stack[i].pos += inc; 23 | if (ctx->stack[i].pos == pg->n_items) { 24 | inc = 1; 25 | if (--i < 0) { 26 | ctx->stack_size = 0; 27 | IMCS_UNLOAD_PAGE(pg); 28 | if (tile_size == 0) { 29 | return false; 30 | } else { 31 | iterator->tile_size = tile_size; 32 | return true; 33 | } 34 | } 35 | } else { 36 | Assert(pg->n_items > ctx->stack[i].pos); 37 | break; 38 | } 39 | IMCS_UNLOAD_PAGE(pg); 40 | } 41 | while (!pg->is_leaf) { 42 | ctx->stack[i+1].page = CHILD(pg, ctx->stack[i].pos).page; 43 | ctx->stack[++i].pos = 0; 44 | IMCS_UNLOAD_PAGE(pg); 45 | pg = ctx->stack[i].page; 46 | IMCS_LOAD_PAGE(pg); 47 | } 48 | Assert(pg->n_items > ctx->stack[i].pos); 49 | n_vals = pg->n_items - ctx->stack[i].pos; 50 | if (n_vals - 1 > iterator->last_pos - iterator->next_pos) { 51 | n_vals = (size_t)(iterator->last_pos - iterator->next_pos + 1); 52 | } 53 | if (n_vals > imcs_tile_size - tile_size) { 54 | n_vals = imcs_tile_size - tile_size; 55 | } 56 | memcpy(&iterator->tile.arr_char[tile_size*iterator->elem_size], &pg->u.val_char[ctx->stack[i].pos*iterator->elem_size], n_vals*iterator->elem_size); 57 | iterator->next_pos += n_vals; 58 | ctx->stack[i].pos += n_vals; 59 | ctx->stack_size = i + 1; 60 | tile_size += n_vals; 61 | IMCS_UNLOAD_PAGE(pg); 62 | if (tile_size == imcs_tile_size || iterator->next_pos > iterator->last_pos) { 63 | iterator->tile_size = tile_size; 64 | return true; 65 | } 66 | } 67 | } 68 | return false; 69 | } 70 | 71 | static bool imcs_next_tile_rle(imcs_iterator_h iterator) 72 | { 73 | imcs_iterator_context_t* ctx = (imcs_iterator_context_t*)iterator->context; 74 | int i = ctx->stack_size-1; 75 | if (i >= 0 && iterator->next_pos <= iterator->last_pos) { 76 | size_t tile_size = 0, j, n_vals; 77 | while (true) { 78 | imcs_page_t* pg; 79 | size_t inc = 0; 80 | while (true) { 81 | pg = ctx->stack[i].page; 82 | IMCS_LOAD_PAGE(pg); 83 | ctx->stack[i].pos += inc; 84 | if (ctx->stack[i].pos == pg->n_items) { 85 | inc = 1; 86 | if (--i < 0) { 87 | ctx->stack_size = 0; 88 | IMCS_UNLOAD_PAGE(pg); 89 | if (tile_size == 0) { 90 | return false; 91 | } else { 92 | iterator->tile_size = tile_size; 93 | return true; 94 | } 95 | } 96 | } else { 97 | Assert(pg->n_items > ctx->stack[i].pos); 98 | break; 99 | } 100 | IMCS_UNLOAD_PAGE(pg); 101 | } 102 | while (!pg->is_leaf) { 103 | ctx->stack[i+1].page = CHILD(pg, ctx->stack[i].pos).page; 104 | ctx->stack[++i].pos = 0; 105 | IMCS_UNLOAD_PAGE(pg); 106 | pg = ctx->stack[i].page; 107 | IMCS_LOAD_PAGE(pg); 108 | } 109 | Assert(pg->n_items > ctx->stack[i].pos); 110 | do { 111 | size_t count = 1 + (pg->u.val_char[ctx->stack[i].pos*(iterator->elem_size+1)] & 0xFF); 112 | bool end = false; 113 | n_vals = count - ctx->rle_offs; 114 | if (n_vals - 1 > iterator->last_pos - iterator->next_pos) { 115 | n_vals = (size_t)(iterator->last_pos - iterator->next_pos + 1); 116 | end = true; 117 | } 118 | if (n_vals > imcs_tile_size - tile_size) { 119 | n_vals = imcs_tile_size - tile_size; 120 | end = true; 121 | } 122 | for (j = 0; j < n_vals; j++) { 123 | memcpy(&iterator->tile.arr_char[(tile_size+j)*iterator->elem_size], &pg->u.val_char[ctx->stack[i].pos*(iterator->elem_size+1)+1], iterator->elem_size); 124 | } 125 | iterator->next_pos += n_vals; 126 | tile_size += n_vals; 127 | if (end) { 128 | ctx->rle_offs += n_vals; 129 | break; 130 | } else { 131 | ctx->rle_offs = 0; 132 | } 133 | } while (++ctx->stack[i].pos < pg->n_items && iterator->next_pos <= iterator->last_pos); 134 | ctx->stack_size = i + 1; 135 | IMCS_UNLOAD_PAGE(pg); 136 | if (tile_size == imcs_tile_size || iterator->next_pos > iterator->last_pos) { 137 | iterator->tile_size = tile_size; 138 | return true; 139 | } 140 | } 141 | } 142 | return false; 143 | } 144 | 145 | 146 | static bool imcs_subseq_page(imcs_iterator_h iterator, imcs_page_t* pg, imcs_pos_t from, int level) 147 | { 148 | imcs_iterator_context_t* ctx = (imcs_iterator_context_t*)iterator->context; 149 | int i, n_items; 150 | Assert(level < IMCS_STACK_SIZE); 151 | ctx->stack[level].page = pg; 152 | IMCS_LOAD_PAGE(pg); 153 | n_items = pg->n_items; 154 | Assert(n_items > 0); 155 | if (!pg->is_leaf) { 156 | for (i = 0; i < n_items; i++) { 157 | imcs_count_t count = CHILD(pg, i).count; 158 | if (from < count) { 159 | imcs_page_t* child = CHILD(pg, i).page; 160 | IMCS_UNLOAD_PAGE(pg); 161 | ctx->stack[level].pos = i; 162 | return imcs_subseq_page(iterator, child, from, level+1); 163 | } 164 | from -= count; 165 | } 166 | } else { 167 | if (iterator->elem_type == TID_char && imcs_use_rle) { 168 | for (i = 0; i < n_items; i++) { 169 | size_t count = 1 + (pg->u.val_char[i*(iterator->elem_size+1)] & 0xFF); 170 | if (from < count) { 171 | IMCS_UNLOAD_PAGE(pg); 172 | ctx->rle_offs = (uint8)from; 173 | ctx->stack[level].pos = i; 174 | ctx->stack_size = level+1; 175 | return true; 176 | } 177 | from -= count; 178 | } 179 | } else { 180 | if (from < (imcs_pos_t)n_items) { 181 | IMCS_UNLOAD_PAGE(pg); 182 | ctx->stack[level].pos = (int)from; 183 | ctx->stack_size = level+1; 184 | return true; 185 | } 186 | } 187 | } 188 | IMCS_UNLOAD_PAGE(pg); 189 | return false; 190 | } 191 | 192 | static void imcs_reset_tree_iterator(imcs_iterator_h iterator) 193 | { 194 | imcs_iterator_context_t* ctx = (imcs_iterator_context_t*)iterator->context; 195 | if (iterator->first_pos <= iterator->last_pos) { 196 | imcs_subseq_page(iterator, ctx->stack[0].page, iterator->first_pos, 0); 197 | } 198 | imcs_reset_iterator(iterator); 199 | } 200 | 201 | void imcs_subseq_random_access_iterator(imcs_iterator_h iterator, imcs_pos_t from, imcs_pos_t till) 202 | { 203 | imcs_timeseries_t* ts = iterator->cs_hdr; 204 | Assert(iterator->last_pos != IMCS_INFINITY); 205 | from = ((int64)from < 0) ? iterator->last_pos + from + 1 : iterator->first_pos + from; 206 | till = ((int64)till < 0) ? iterator->last_pos + till + 1 : iterator->first_pos + till; 207 | if (from < iterator->first_pos) { 208 | from = iterator->first_pos; 209 | } 210 | if (till > iterator->last_pos) { 211 | till = iterator->last_pos; 212 | } 213 | if (till >= from && (ts == NULL || (ts->root_page != NULL && imcs_subseq_page(iterator, ts->root_page, from, 0)))) { 214 | iterator->first_pos = iterator->next_pos = from; 215 | iterator->last_pos = till; 216 | } else { 217 | iterator->first_pos = iterator->next_pos = 1; 218 | iterator->last_pos = 0; 219 | } 220 | } 221 | 222 | imcs_iterator_h imcs_subseq(imcs_timeseries_t* ts, imcs_pos_t from, imcs_pos_t till) 223 | { 224 | imcs_iterator_t* iterator; 225 | int elem_size = ts->elem_size; 226 | int elem_type = ts->elem_type; 227 | int flags = FLAG_CONTEXT_FREE|FLAG_RANDOM_ACCESS; 228 | if (elem_size < 0) { /* varying string */ 229 | Assert(elem_type == TID_char); 230 | flags |= FLAG_TRANSLATED; 231 | if (imcs_dict_size <= IMCS_SMALL_DICTIONARY) { 232 | elem_size = 2; 233 | elem_type = TID_int16; 234 | } else { 235 | elem_size = 4; 236 | elem_type = TID_int32; 237 | } 238 | } 239 | iterator = (imcs_iterator_t*)imcs_new_iterator(elem_size, sizeof(imcs_iterator_context_t)); 240 | iterator->cs_hdr = ts; 241 | iterator->elem_type = elem_type; 242 | iterator->flags = flags; 243 | iterator->reset = imcs_reset_tree_iterator; 244 | iterator->next = (elem_type == TID_char && imcs_use_rle) ? imcs_next_tile_rle : imcs_next_tile; 245 | if (till >= from && ts->root_page != NULL && imcs_subseq_page(iterator, ts->root_page, from, 0)) { 246 | iterator->first_pos = iterator->next_pos = from; 247 | iterator->last_pos = till >= ts->count ? ts->count-1 : till; 248 | } else { 249 | iterator->first_pos = iterator->next_pos = 1; 250 | iterator->last_pos = 0; 251 | } 252 | return iterator; 253 | } 254 | 255 | static bool imcs_map_next(imcs_iterator_h iterator) 256 | { 257 | size_t i, tile_size; 258 | imcs_iterator_context_t* ctx = (imcs_iterator_context_t*)iterator->context; 259 | size_t elem_size = iterator->elem_size; 260 | size_t prev_page_size = 0; 261 | imcs_pos_t prev_page_pos = 0; 262 | imcs_iterator_h map = iterator->opd[0]; 263 | 264 | if (!map->next(map)) { 265 | return false; 266 | } 267 | tile_size = map->tile_size; 268 | for (i = 0; i < tile_size; i++) { 269 | imcs_pos_t next_pos = iterator->first_pos + map->tile.arr_int64[i]; 270 | imcs_page_t* pg; 271 | if (next_pos - prev_page_pos >= prev_page_size) { 272 | if (!imcs_subseq_page(iterator, ctx->stack[0].page, next_pos, 0)) { 273 | imcs_ereport(ERRCODE_INVALID_PARAMETER_VALUE, "invalid position in timeseries"); 274 | } 275 | prev_page_pos = next_pos - ctx->stack[ctx->stack_size-1].pos; 276 | } 277 | pg = ctx->stack[ctx->stack_size-1].page; 278 | IMCS_LOAD_PAGE(pg); 279 | prev_page_size = pg->n_items; 280 | Assert(next_pos - prev_page_pos < prev_page_size); 281 | memcpy(&iterator->tile.arr_char[i*elem_size], &pg->u.val_char[((size_t)(next_pos - prev_page_pos))*elem_size], elem_size); 282 | IMCS_UNLOAD_PAGE(pg); 283 | } 284 | iterator->next_pos += tile_size; 285 | iterator->tile_size = tile_size; 286 | return true; 287 | } 288 | 289 | static bool imcs_map_next_rle(imcs_iterator_h iterator) 290 | { 291 | size_t i, tile_size; 292 | imcs_iterator_context_t* ctx = (imcs_iterator_context_t*)iterator->context; 293 | size_t elem_size = iterator->elem_size; 294 | imcs_iterator_h map = iterator->opd[0]; 295 | 296 | if (!map->next(map)) { 297 | return false; 298 | } 299 | tile_size = map->tile_size; 300 | for (i = 0; i < tile_size; i++) { 301 | imcs_pos_t next_pos = iterator->first_pos + map->tile.arr_int64[i]; 302 | imcs_page_t* pg; 303 | if (!imcs_subseq_page(iterator, ctx->stack[0].page, next_pos, 0)) { 304 | imcs_ereport(ERRCODE_INVALID_PARAMETER_VALUE, "invalid position in timeseries"); 305 | } 306 | pg = ctx->stack[ctx->stack_size-1].page; 307 | IMCS_LOAD_PAGE(pg); 308 | memcpy(&iterator->tile.arr_char[i*elem_size], &pg->u.val_char[ctx->stack[ctx->stack_size-1].pos*(elem_size+1)+1], elem_size); 309 | IMCS_UNLOAD_PAGE(pg); 310 | } 311 | iterator->next_pos += tile_size; 312 | iterator->tile_size = tile_size; 313 | return true; 314 | } 315 | 316 | imcs_iterator_h imcs_map(imcs_iterator_h input, imcs_iterator_h map_iterator) 317 | { 318 | imcs_timeseries_t* ts = input->cs_hdr; 319 | int elem_size = ts->elem_size; 320 | int elem_type = ts->elem_type; 321 | int flags = FLAG_CONTEXT_FREE; 322 | imcs_iterator_h iterator; 323 | imcs_iterator_context_t* ctx; 324 | 325 | if (elem_size < 0) { /* varying string */ 326 | Assert(elem_type == TID_char); 327 | flags |= FLAG_TRANSLATED; 328 | if (imcs_dict_size <= IMCS_SMALL_DICTIONARY) { 329 | elem_size = 2; 330 | elem_type = TID_int16; 331 | } else { 332 | elem_size = 4; 333 | elem_type = TID_int32; 334 | } 335 | } 336 | iterator = imcs_new_iterator(elem_size, sizeof(imcs_iterator_context_t)); 337 | ctx = (imcs_iterator_context_t*)iterator->context; 338 | iterator->elem_type = elem_type; 339 | iterator->flags = flags; 340 | iterator->next = (elem_type == TID_char && imcs_use_rle) ? imcs_map_next_rle : imcs_map_next; 341 | iterator->opd[0] = map_iterator; 342 | iterator->first_pos = iterator->next_pos = input->first_pos; 343 | ctx->stack[0].page = ts->root_page; 344 | return iterator; 345 | } 346 | 347 | 348 | #define IMCS_IMPLEMENTATIONS(TYPE) \ 349 | bool imcs_first_##TYPE(imcs_timeseries_t* ts, TYPE* val) \ 350 | { \ 351 | imcs_page_t* pg = ts->root_page; \ 352 | if (pg == NULL) { \ 353 | return false; \ 354 | } \ 355 | IMCS_LOAD_PAGE(pg); \ 356 | while (!pg->is_leaf) { \ 357 | imcs_page_t* child = CHILD(pg, 0).page; \ 358 | IMCS_UNLOAD_PAGE(pg); \ 359 | pg = child; \ 360 | IMCS_LOAD_PAGE(pg); \ 361 | } \ 362 | *val = pg->u.val_##TYPE[0]; \ 363 | IMCS_UNLOAD_PAGE(pg); \ 364 | return true; \ 365 | } \ 366 | bool imcs_last_##TYPE(imcs_timeseries_t* ts, TYPE* val) \ 367 | { \ 368 | imcs_page_t* pg = ts->root_page; \ 369 | if (pg == NULL) { \ 370 | return false; \ 371 | } \ 372 | IMCS_LOAD_PAGE(pg); \ 373 | while (!pg->is_leaf) { \ 374 | imcs_page_t* child = CHILD(pg, pg->n_items-1).page; \ 375 | IMCS_UNLOAD_PAGE(pg); \ 376 | pg = child; \ 377 | IMCS_LOAD_PAGE(pg); \ 378 | } \ 379 | *val = pg->u.val_##TYPE[pg->n_items-1]; \ 380 | IMCS_UNLOAD_PAGE(pg); \ 381 | return true; \ 382 | } \ 383 | /* return false on overflow */ \ 384 | static bool imcs_append_page_##TYPE(imcs_page_t** root_page, TYPE val, bool is_timestamp) \ 385 | { \ 386 | imcs_page_t* pg = *root_page; \ 387 | int n_items; \ 388 | IMCS_LOAD_PAGE_FOR_UPDATE(pg); \ 389 | n_items = pg->n_items; \ 390 | Assert(n_items > 0); \ 391 | if (is_timestamp && pg->u.val_##TYPE[n_items-1] > val) { \ 392 | IMCS_UNLOAD_PAGE(pg); \ 393 | imcs_ereport(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE, "value out of timeseries order"); \ 394 | } \ 395 | if (!pg->is_leaf) { \ 396 | imcs_page_t* child = CHILD(pg, n_items-1).page; \ 397 | if (!imcs_append_page_##TYPE(&child, val, is_timestamp)) { \ 398 | if (n_items == MAX_NODE_ITEMS(TYPE)) { \ 399 | imcs_page_t* new_page = imcs_new_page(); \ 400 | *root_page = new_page; \ 401 | IMCS_UNLOAD_PAGE(pg); \ 402 | IMCS_LOAD_NEW_PAGE(new_page); \ 403 | new_page->is_leaf = false; \ 404 | new_page->n_items = 1; \ 405 | CHILD(new_page, 0).page = child; \ 406 | CHILD(new_page, 0).count = 1; \ 407 | if (is_timestamp) { \ 408 | new_page->u.val_##TYPE[0] = val; \ 409 | } \ 410 | IMCS_UNLOAD_PAGE(new_page); \ 411 | return false; \ 412 | } else { \ 413 | CHILD(pg, n_items).page = child; \ 414 | CHILD(pg, n_items).count = 1; \ 415 | if (is_timestamp) { \ 416 | pg->u.val_##TYPE[n_items] = val; \ 417 | } \ 418 | pg->n_items += 1; \ 419 | } \ 420 | } else { \ 421 | CHILD(pg, n_items-1).count += 1; \ 422 | } \ 423 | } else { \ 424 | int max_items = MAX_LEAF_ITEMS(TYPE); \ 425 | Assert(n_items <= max_items); \ 426 | if (n_items == max_items) { \ 427 | imcs_page_t* new_page = imcs_new_page(); \ 428 | *root_page = new_page; \ 429 | IMCS_UNLOAD_PAGE(pg); \ 430 | IMCS_LOAD_NEW_PAGE(new_page); \ 431 | new_page->is_leaf = true; \ 432 | new_page->u.val_##TYPE[0] = val; \ 433 | new_page->n_items = 1; \ 434 | IMCS_UNLOAD_PAGE(new_page); \ 435 | return false; \ 436 | } else { \ 437 | pg->u.val_##TYPE[n_items] = val; \ 438 | pg->n_items += 1; \ 439 | } \ 440 | } \ 441 | IMCS_UNLOAD_PAGE(pg); \ 442 | return true; \ 443 | } \ 444 | \ 445 | void imcs_append_##TYPE(imcs_timeseries_t* ts, TYPE val) \ 446 | { \ 447 | if (ts->root_page == 0) { \ 448 | imcs_page_t* pg = imcs_new_page(); \ 449 | ts->root_page = pg; \ 450 | IMCS_LOAD_NEW_PAGE(pg); \ 451 | pg->is_leaf = true; \ 452 | pg->u.val_##TYPE[0] = val; \ 453 | ts->count = pg->n_items = 1; \ 454 | IMCS_UNLOAD_PAGE(pg); \ 455 | } else { \ 456 | imcs_page_t* root_page = ts->root_page; \ 457 | imcs_page_t* old_root = root_page; \ 458 | if (!imcs_append_page_##TYPE(&root_page, val, ts->is_timestamp)) { \ 459 | imcs_page_t* new_root = imcs_new_page(); \ 460 | ts->root_page = new_root; \ 461 | IMCS_LOAD_NEW_PAGE(new_root); \ 462 | new_root->is_leaf = false; \ 463 | new_root->n_items = 2; \ 464 | CHILD(new_root, 0).page = old_root; \ 465 | CHILD(new_root, 0).count = ts->count; \ 466 | CHILD(new_root, 1).page = root_page; \ 467 | CHILD(new_root, 1).count = 1; \ 468 | if (ts->is_timestamp) { \ 469 | IMCS_LOAD_PAGE(old_root); \ 470 | new_root->u.val_##TYPE[0] = old_root->u.val_##TYPE[0]; \ 471 | new_root->u.val_##TYPE[1] = val; \ 472 | IMCS_UNLOAD_PAGE(old_root); \ 473 | } \ 474 | IMCS_UNLOAD_PAGE(new_root); \ 475 | } \ 476 | ts->count += 1; \ 477 | } \ 478 | } \ 479 | bool imcs_search_page_##TYPE(imcs_page_t* pg, imcs_iterator_h iterator, TYPE val, imcs_boundary_kind_t boundary, int level) \ 480 | { \ 481 | int i, l, r, n_items; \ 482 | imcs_iterator_context_t* ctx = (imcs_iterator_context_t*)iterator->context; \ 483 | bool found = false; \ 484 | Assert(level < IMCS_STACK_SIZE); \ 485 | ctx->stack[level].page = pg; \ 486 | IMCS_LOAD_PAGE(pg); \ 487 | n_items = pg->n_items; \ 488 | Assert(n_items > 0); \ 489 | l = 0; \ 490 | r = n_items; \ 491 | if (boundary == BOUNDARY_INCLUSIVE || boundary == BOUNDARY_EXACT) { \ 492 | while (l < r) { \ 493 | int m = (l + r) >> 1; \ 494 | if (pg->u.val_##TYPE[m] < val) { \ 495 | l = m + 1; \ 496 | } else { \ 497 | r = m; \ 498 | } \ 499 | } \ 500 | } else if (boundary == BOUNDARY_EXCLUSIVE) { \ 501 | while (l < r) { \ 502 | int m = (l + r) >> 1; \ 503 | if (pg->u.val_##TYPE[m] <= val) { \ 504 | l = m + 1; \ 505 | } else { \ 506 | r = m; \ 507 | } \ 508 | } \ 509 | } \ 510 | if (!pg->is_leaf) { \ 511 | if (l > 0) { \ 512 | found = imcs_search_page_##TYPE(CHILD(pg, l-1).page, iterator, val, boundary, level+1); \ 513 | if (found) { \ 514 | l -= 1; \ 515 | } \ 516 | } \ 517 | if (!found && l < n_items) { \ 518 | found = imcs_search_page_##TYPE(CHILD(pg, l).page, iterator, val, boundary, level+1); \ 519 | } \ 520 | if (found) { \ 521 | for (i = 0; i < l; i++) { \ 522 | iterator->next_pos += CHILD(pg, i).count; \ 523 | } \ 524 | ctx->stack[level].pos = l; \ 525 | } \ 526 | } else { \ 527 | if (l < n_items && (boundary != BOUNDARY_EXACT || pg->u.val_##TYPE[l] == val)) { \ 528 | iterator->next_pos += l; \ 529 | ctx->stack[level].pos = l; \ 530 | ctx->stack_size = level+1; \ 531 | found = true; \ 532 | } \ 533 | } \ 534 | IMCS_UNLOAD_PAGE(pg); \ 535 | return found; \ 536 | } \ 537 | \ 538 | imcs_iterator_h imcs_search_##TYPE(imcs_timeseries_t* ts, TYPE low, imcs_boundary_kind_t low_boundary, TYPE high, imcs_boundary_kind_t high_boundary, imcs_count_t limit) \ 539 | { \ 540 | imcs_iterator_h iterator = NULL; \ 541 | if (ts->root_page != NULL) { \ 542 | iterator = imcs_new_iterator(sizeof(TYPE), sizeof(imcs_iterator_context_t)); \ 543 | iterator->reset = imcs_reset_tree_iterator; \ 544 | iterator->next = imcs_next_tile; \ 545 | iterator->elem_type = ts->elem_type; \ 546 | iterator->cs_hdr = ts; \ 547 | iterator->flags = FLAG_CONTEXT_FREE|FLAG_RANDOM_ACCESS; \ 548 | if (high_boundary != BOUNDARY_OPEN) { \ 549 | iterator->next_pos = 0; \ 550 | if (!imcs_search_page_##TYPE(ts->root_page, iterator, high, BOUNDARY_INCLUSIVE + BOUNDARY_EXCLUSIVE - high_boundary, 0)) { \ 551 | iterator->last_pos = ts->count-1; \ 552 | } else { \ 553 | if (iterator->next_pos == 0) { \ 554 | iterator->first_pos = iterator->next_pos = 1; \ 555 | iterator->last_pos = 0; \ 556 | return iterator; \ 557 | } else { \ 558 | iterator->last_pos = iterator->next_pos - 1; \ 559 | } \ 560 | } \ 561 | } else { \ 562 | iterator->last_pos = ts->count-1; \ 563 | } \ 564 | iterator->next_pos = 0; \ 565 | if (imcs_search_page_##TYPE(ts->root_page, iterator, low, low_boundary, 0)) { \ 566 | if (iterator->next_pos <= iterator->last_pos) { \ 567 | if (limit != 0 && iterator->next_pos + limit <= iterator->last_pos) { \ 568 | if (low_boundary == BOUNDARY_OPEN) { \ 569 | iterator->next_pos = iterator->last_pos - limit + 1; \ 570 | } else { \ 571 | iterator->last_pos = iterator->next_pos + limit - 1; \ 572 | } \ 573 | } \ 574 | iterator->first_pos = iterator->next_pos; \ 575 | return iterator; \ 576 | } \ 577 | } \ 578 | iterator->first_pos = iterator->next_pos = 1; \ 579 | iterator->last_pos = 0; \ 580 | } \ 581 | return iterator; \ 582 | } \ 583 | 584 | IMCS_IMPLEMENTATIONS(int8) 585 | IMCS_IMPLEMENTATIONS(int16) 586 | IMCS_IMPLEMENTATIONS(int32) 587 | IMCS_IMPLEMENTATIONS(int64) 588 | IMCS_IMPLEMENTATIONS(float) 589 | IMCS_IMPLEMENTATIONS(double) 590 | 591 | static bool imcs_append_page_char(imcs_page_t** root_page, char const* val, size_t val_len, size_t elem_size) 592 | { 593 | imcs_page_t* pg = *root_page; 594 | int n_items; 595 | IMCS_LOAD_PAGE_FOR_UPDATE(pg); 596 | n_items = pg->n_items; 597 | Assert(n_items > 0); 598 | if (!pg->is_leaf) { 599 | imcs_page_t* child = CHILD(pg, n_items-1).page; 600 | if (!imcs_append_page_char(&child, val, val_len, elem_size)) { 601 | if (n_items == MAX_NODE_ITEMS_CHAR()) { 602 | imcs_page_t* new_page = imcs_new_page(); 603 | *root_page = new_page; 604 | IMCS_UNLOAD_PAGE(pg); 605 | IMCS_LOAD_NEW_PAGE(new_page); 606 | new_page->is_leaf = false; 607 | new_page->n_items = 1; 608 | CHILD(new_page, 0).page = child; 609 | CHILD(new_page, 0).count = 1; 610 | IMCS_UNLOAD_PAGE(new_page); 611 | return false; 612 | } else { 613 | CHILD(pg, n_items).page = child; 614 | CHILD(pg, n_items).count = 1; 615 | pg->n_items += 1; 616 | } 617 | } else { 618 | CHILD(pg, n_items-1).count += 1; 619 | } 620 | } else { 621 | int max_items = MAX_LEAF_ITEMS_CHAR(elem_size); 622 | Assert(n_items <= max_items); 623 | if (n_items == max_items) { 624 | imcs_page_t* new_page = imcs_new_page(); 625 | *root_page = new_page; 626 | IMCS_UNLOAD_PAGE(pg); 627 | IMCS_LOAD_NEW_PAGE(new_page); 628 | new_page->is_leaf = true; 629 | memcpy(new_page->u.val_char, val, val_len); 630 | memset(new_page->u.val_char + val_len, '\0', elem_size - val_len); 631 | new_page->n_items = 1; 632 | IMCS_UNLOAD_PAGE(new_page); 633 | return false; 634 | } else { 635 | memcpy(&pg->u.val_char[n_items*elem_size], val, val_len); 636 | memset(&pg->u.val_char[n_items*elem_size + val_len], '\0', elem_size - val_len); 637 | pg->n_items += 1; 638 | } 639 | } 640 | IMCS_UNLOAD_PAGE(pg); 641 | return true; 642 | } 643 | 644 | static bool imcs_append_page_char_rle(imcs_page_t** root_page, char const* val, size_t val_len, size_t elem_size) 645 | { 646 | imcs_page_t* pg = *root_page; 647 | int n_items; 648 | IMCS_LOAD_PAGE_FOR_UPDATE(pg); 649 | n_items = pg->n_items; 650 | Assert(n_items > 0); 651 | if (!pg->is_leaf) { 652 | imcs_page_t* child = CHILD(pg, n_items-1).page; 653 | if (!imcs_append_page_char_rle(&child, val, val_len, elem_size)) { 654 | if (n_items == MAX_NODE_ITEMS_CHAR()) { 655 | imcs_page_t* new_page = imcs_new_page(); 656 | *root_page = new_page; 657 | IMCS_UNLOAD_PAGE(pg); 658 | IMCS_LOAD_NEW_PAGE(new_page); 659 | new_page->is_leaf = false; 660 | new_page->n_items = 1; 661 | CHILD(new_page, 0).page = child; 662 | CHILD(new_page, 0).count = 1; 663 | IMCS_UNLOAD_PAGE(new_page); 664 | return false; 665 | } else { 666 | CHILD(pg, n_items).page = child; 667 | CHILD(pg, n_items).count = 1; 668 | pg->n_items += 1; 669 | } 670 | } else { 671 | CHILD(pg, n_items-1).count += 1; 672 | } 673 | } else { 674 | if (pg->u.val_char[(n_items-1)*(elem_size+1)] != -1 && memcmp(&pg->u.val_char[(n_items-1)*(elem_size+1)+1], val, val_len) == 0 675 | && (val_len == elem_size || pg->u.val_char[(n_items-1)*(elem_size+1)+1+val_len] == '\0')) 676 | { 677 | pg->u.val_char[(n_items-1)*(elem_size+1)] += 1; 678 | } else { 679 | int max_items = MAX_LEAF_ITEMS_CHAR(elem_size+1); 680 | Assert(n_items <= max_items); 681 | if (n_items == max_items) { 682 | imcs_page_t* new_page = imcs_new_page(); 683 | *root_page = new_page; 684 | IMCS_UNLOAD_PAGE(pg); 685 | IMCS_LOAD_NEW_PAGE(new_page); 686 | new_page->is_leaf = true; 687 | new_page->u.val_char[0] = 0; 688 | memcpy(new_page->u.val_char + 1, val, val_len); 689 | memset(new_page->u.val_char + val_len + 1, '\0', elem_size - val_len); 690 | new_page->n_items = 1; 691 | IMCS_UNLOAD_PAGE(new_page); 692 | return false; 693 | } else { 694 | pg->u.val_char[n_items*(elem_size+1)] = 0; 695 | memcpy(&pg->u.val_char[n_items*(elem_size+1) + 1], val, val_len); 696 | memset(&pg->u.val_char[n_items*(elem_size+1) + 1 + val_len], '\0', elem_size - val_len); 697 | pg->n_items += 1; 698 | } 699 | } 700 | } 701 | IMCS_UNLOAD_PAGE(pg); 702 | return true; 703 | } 704 | 705 | 706 | void imcs_append_char(imcs_timeseries_t* ts, char const* val, size_t val_len) 707 | { 708 | Assert(!ts->is_timestamp); 709 | Assert(val_len <= ts->elem_size); 710 | if (ts->root_page == 0) { 711 | imcs_page_t* pg = imcs_new_page(); 712 | char* dst; 713 | ts->root_page = pg; 714 | IMCS_LOAD_NEW_PAGE(pg); 715 | dst = pg->u.val_char; 716 | pg->is_leaf = true; 717 | if (imcs_use_rle) { 718 | *dst++ = 0; 719 | } 720 | memcpy(dst, val, val_len); 721 | memset(dst + val_len, 0, ts->elem_size - val_len); 722 | ts->count = pg->n_items = 1; 723 | IMCS_UNLOAD_PAGE(pg); 724 | } else { 725 | imcs_page_t* root_page = ts->root_page; 726 | imcs_page_t* old_root = root_page; 727 | bool no_overflow = imcs_use_rle 728 | ? imcs_append_page_char_rle(&root_page, val, val_len, ts->elem_size) 729 | : imcs_append_page_char(&root_page, val, val_len, ts->elem_size); 730 | if (!no_overflow) { 731 | imcs_page_t* new_root = imcs_new_page(); 732 | ts->root_page = new_root; 733 | IMCS_LOAD_NEW_PAGE(new_root); 734 | new_root->is_leaf = false; 735 | new_root->n_items = 2; 736 | CHILD(new_root, 0).page = old_root; 737 | CHILD(new_root, 0).count = ts->count; 738 | CHILD(new_root, 1).page = root_page; 739 | CHILD(new_root, 1).count = 1; 740 | IMCS_UNLOAD_PAGE(new_root); 741 | } 742 | ts->count += 1; 743 | } 744 | } 745 | 746 | /* returns false on underflow */ 747 | static bool imcs_delete_page(imcs_timeseries_t* ts, imcs_page_t* pg, imcs_pos_t from, imcs_pos_t till) 748 | { 749 | int n_items; 750 | int elem_size = ts->elem_size; 751 | int elem_type = ts->elem_type; 752 | int i; 753 | if (elem_size < 0) { /* varying string */ 754 | Assert(elem_type == TID_char); 755 | if (imcs_dict_size <= IMCS_SMALL_DICTIONARY) { 756 | elem_size = 2; 757 | elem_type = TID_int16; 758 | } else { 759 | elem_size = 4; 760 | elem_type = TID_int32; 761 | } 762 | } 763 | IMCS_LOAD_PAGE_FOR_UPDATE(pg); 764 | n_items = pg->n_items; 765 | if (!pg->is_leaf) { 766 | for (i = 0; i < n_items; i++) { 767 | imcs_count_t count = CHILD(pg, i).count; 768 | if (from < count) { 769 | int j = i; 770 | do { 771 | count = CHILD(pg, i).count; 772 | if (imcs_delete_page(ts, CHILD(pg, i).page, from, till)) { 773 | if (till < count) { 774 | CHILD(pg, i).count -= till - from + 1; 775 | break; 776 | } 777 | Assert(from != 0); 778 | CHILD(pg, i).count = from; 779 | j = i + 1; 780 | from = 0; 781 | } else { 782 | Assert(from == 0); 783 | if (till < count) { 784 | i += 1; 785 | break; 786 | } 787 | } 788 | till -= count; 789 | } while (++i < n_items); 790 | 791 | if (i != j) { 792 | MOVE_CHILDREN(pg, j, pg, i, n_items-i); 793 | if (ts->is_timestamp) { 794 | memmove(&pg->u.val_char[j*elem_size], &pg->u.val_char[i*elem_size], (n_items-i)*elem_size); 795 | } 796 | pg->n_items -= i - j; 797 | if (pg->n_items == 0) { 798 | imcs_free_page(pg); 799 | return false; 800 | } 801 | } 802 | break; 803 | } else { 804 | from -= count; 805 | till -= count; 806 | } 807 | } 808 | } else { 809 | if (elem_type == TID_char && imcs_use_rle) { 810 | for (i = 0; i < n_items; i++) { 811 | size_t count = 1 + (pg->u.val_char[i*(elem_size+1)] & 0xFF); 812 | if (from < count) { 813 | if (till < count-1) { /* all deleted elements have the same value */ 814 | pg->u.val_char[i*(elem_size+1)] = (char)(count - till + from - 2); 815 | ts->count -= till - from + 1; 816 | } else { 817 | int delete_from; 818 | if (from != 0) { /* first value is not completely deleted */ 819 | ts->count += from; 820 | pg->u.val_char[i*(elem_size+1)] = (char)(from - 1); 821 | delete_from = i+1; /* index of first deleted element on the page */ 822 | } else { 823 | delete_from = i; /* index of first deleted element on the page */ 824 | } 825 | while (till >= count && ++i < n_items) { 826 | ts->count -= count; 827 | till -= count; 828 | count = 1 + (pg->u.val_char[i*(elem_size+1)] & 0xFF); 829 | } 830 | if (till == count-1) { /* last value is completely deleted */ 831 | ts->count -= count; 832 | i += 1; 833 | } else if (i < n_items) { 834 | ts->count -= till+1; 835 | pg->u.val_char[i*(elem_size+1)] = (char)(count - till - 2); 836 | } else { 837 | ts->count -= count; 838 | } 839 | if (i-delete_from == n_items) { 840 | imcs_free_page(pg); 841 | return false; 842 | } else { 843 | memmove(&pg->u.val_char[delete_from*(elem_size+1)], &pg->u.val_char[i*(elem_size+1)], (n_items-i)*(elem_size+1)); 844 | pg->n_items -= i - delete_from; 845 | } 846 | } 847 | break; 848 | } else { 849 | from -= count; 850 | till -= count; 851 | } 852 | } 853 | } else { 854 | if (till >= (imcs_pos_t)n_items) { 855 | till = n_items-1; 856 | } 857 | ts->count -= till - from + 1; 858 | if ((int)(till - from + 1) == n_items) { 859 | imcs_free_page(pg); 860 | return false; 861 | } else { 862 | memmove(&pg->u.val_char[from*elem_size], &pg->u.val_char[(till+1)*elem_size], (n_items-till-1)*elem_size); 863 | pg->n_items -= till - from + 1; 864 | } 865 | } 866 | } 867 | IMCS_UNLOAD_PAGE(pg); 868 | return true; 869 | } 870 | 871 | void imcs_delete(imcs_timeseries_t* ts, imcs_pos_t from, imcs_pos_t till) 872 | { 873 | imcs_page_t* root_page = ts->root_page; 874 | if (root_page != NULL && from <= till) { 875 | if (!imcs_delete_page(ts, root_page, from, till)) { 876 | Assert(ts->count == 0); 877 | ts->root_page = NULL; 878 | } 879 | } 880 | } 881 | 882 | static void imcs_prune(imcs_page_t* pg) 883 | { 884 | IMCS_LOAD_PAGE(pg); 885 | if (!pg->is_leaf) { 886 | int i, n; 887 | for (i = 0, n = pg->n_items; i < n; i++) { 888 | imcs_prune(CHILD(pg, i).page); 889 | } 890 | } 891 | imcs_free_page(pg); 892 | } 893 | 894 | imcs_count_t imcs_delete_all(imcs_timeseries_t* ts) 895 | { 896 | imcs_page_t* root_page = ts->root_page; 897 | imcs_count_t count = ts->count; 898 | if (root_page != NULL) { 899 | imcs_prune(root_page); 900 | ts->root_page = NULL; 901 | } 902 | ts->count = 0; 903 | return count; 904 | } 905 | 906 | -------------------------------------------------------------------------------- /btree.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Implementation of B-Tree storing timeseries data 3 | */ 4 | #ifndef __BTREE_H__ 5 | #define __BTREE_H__ 6 | 7 | #include "imcs.h" 8 | 9 | #define OFFSETOF_ITEMS(TYPE) ((size_t)((imcs_page_t*)0)->u.val_##TYPE) 10 | #define CHILD(pg, i) (*(imcs_node_t*)((char*)(pg) + imcs_page_size - sizeof(imcs_node_t)*((i) + 1))) 11 | #define MAX_LEAF_ITEMS(TYPE) ((int)((imcs_page_size - OFFSETOF_ITEMS(TYPE))/sizeof(TYPE))) 12 | #define MAX_NODE_ITEMS(TYPE) ((int)((imcs_page_size - OFFSETOF_ITEMS(TYPE))/(!is_timestamp ? sizeof(imcs_node_t) : sizeof(TYPE) + sizeof(imcs_node_t)))) 13 | 14 | #define MOVE_CHILDREN(dst_page, dst_index, src_page, src_index, size) memmove(&CHILD(dst_page, (dst_index) + (size) - 1), &CHILD(src_page, (src_index) + (size) - 1), (size)*sizeof(imcs_node_t)) 15 | 16 | #define OFFSETOF_ITEMS_CHAR ((size_t)((imcs_page_t*)0)->u.val_char) 17 | #define MAX_LEAF_ITEMS_CHAR(size) ((int)((imcs_page_size - OFFSETOF_ITEMS_CHAR)/(size))) 18 | #define MAX_NODE_ITEMS_CHAR() ((int)((imcs_page_size - OFFSETOF_ITEMS_CHAR)/sizeof(imcs_node_t))) 19 | 20 | typedef struct imcs_node_t { 21 | struct imcs_page_t_* page; 22 | uint64 count; 23 | } imcs_node_t; 24 | 25 | struct imcs_page_t_ { 26 | uint32 n_items : 31; 27 | uint32 is_leaf : 1; 28 | union { 29 | char val_char[2]; 30 | int8 val_int8[2]; 31 | int16 val_int16[2]; 32 | int32 val_int32[2]; 33 | int64 val_int64[2]; 34 | float val_float[2]; 35 | double val_double[2]; 36 | imcs_node_t child[2]; /* filled from the end of the page: child[imcs_page_size / sizeof(imcs_node_t) - index] */ 37 | } u; 38 | }; 39 | 40 | typedef struct imcs_iterator_stack_item_t_ { 41 | imcs_page_t* page; 42 | int pos; 43 | } imcs_iterator_stack_item_t; 44 | 45 | #define IMCS_STACK_SIZE 16 46 | 47 | typedef struct imcs_iterator_context_t_ 48 | { 49 | uint8 stack_size; 50 | int8 direction; /* used for timestamp join */ 51 | uint8 rle_offs; /* offset within duplicate values for RLE encoding */ 52 | imcs_iterator_stack_item_t stack[IMCS_STACK_SIZE]; 53 | } imcs_iterator_context_t; 54 | 55 | typedef enum { 56 | BOUNDARY_OPEN, 57 | BOUNDARY_INCLUSIVE, 58 | BOUNDARY_EXCLUSIVE, 59 | BOUNDARY_EXACT 60 | } imcs_boundary_kind_t; 61 | 62 | extern void imcs_subseq_random_access_iterator(imcs_iterator_h iterator, imcs_pos_t from, imcs_pos_t till); 63 | 64 | extern imcs_iterator_h imcs_subseq(imcs_timeseries_t* ts, imcs_pos_t from, imcs_pos_t till); 65 | extern imcs_iterator_h imcs_map(imcs_iterator_h ts, imcs_iterator_h map_iterator); 66 | 67 | extern void imcs_delete(imcs_timeseries_t* ts, imcs_pos_t from, imcs_pos_t till); 68 | extern imcs_count_t imcs_delete_all(imcs_timeseries_t* ts); 69 | 70 | #define IMCS_BTREE_METHODS(TYPE) \ 71 | extern void imcs_append_##TYPE(imcs_timeseries_t* ts, TYPE val); \ 72 | extern bool imcs_first_##TYPE(imcs_timeseries_t* ts, TYPE* val); \ 73 | extern bool imcs_last_##TYPE(imcs_timeseries_t* ts, TYPE* val); \ 74 | extern imcs_iterator_h imcs_search_##TYPE(imcs_timeseries_t* ts, TYPE low, imcs_boundary_kind_t low_boundary, TYPE high, imcs_boundary_kind_t high_boundary, imcs_count_t limit); \ 75 | extern bool imcs_search_page_##TYPE(imcs_page_t* root, imcs_iterator_h iterator, TYPE val, imcs_boundary_kind_t boundary, int level) 76 | 77 | IMCS_BTREE_METHODS(int8); 78 | IMCS_BTREE_METHODS(int16); 79 | IMCS_BTREE_METHODS(int32); 80 | IMCS_BTREE_METHODS(int64); 81 | IMCS_BTREE_METHODS(float); 82 | IMCS_BTREE_METHODS(double); 83 | 84 | extern void imcs_append_char(imcs_timeseries_t* ts, char const* val, size_t val_len); 85 | 86 | #endif 87 | -------------------------------------------------------------------------------- /disk.c: -------------------------------------------------------------------------------- 1 | #include "disk.h" 2 | #include "btree.h" 3 | #include "fileio.h" 4 | 5 | static imcs_file_h imcs_file; 6 | static imcs_disk_cache_t* imcs_disk_cache; 7 | 8 | inline static void imcs_unlink(int pid) 9 | { 10 | imcs_disk_cache_t* cache = imcs_disk_cache; 11 | imcs_cache_item_t* item = &cache->items[pid]; 12 | 13 | cache->items[item->prev].next = item->next; 14 | cache->items[item->next].prev = item->prev; 15 | 16 | /* adjust if needed pointer of last internal page in LRU list */ 17 | if (cache->lru_internal == pid) { 18 | cache->lru_internal = item->prev; 19 | } 20 | } 21 | 22 | inline static void imcs_link_after(int after, int pid) 23 | { 24 | imcs_disk_cache_t* cache = imcs_disk_cache; 25 | imcs_cache_item_t* item = &cache->items[pid]; 26 | 27 | cache->items[item->next = cache->items[after].next].prev = pid; 28 | cache->items[item->prev = after].next = pid; 29 | } 30 | 31 | imcs_page_t* imcs_load_page(imcs_page_t* pg, imcs_page_access_mode_t mode) 32 | { 33 | size_t offs = (size_t)pg; 34 | size_t h = (offs / imcs_page_size) % imcs_cache_size; 35 | size_t pid; 36 | imcs_cache_item_t* item; 37 | imcs_disk_cache_t* cache = imcs_disk_cache; 38 | 39 | Retry: 40 | SpinLockAcquire(&cache->mutex); 41 | for (pid = cache->hash_table[h]; pid != 0; pid = item->collision) { 42 | item = &cache->items[pid]; 43 | if (item->offs == offs) { 44 | while (item->is_busy) { 45 | SpinLockRelease(&cache->mutex); 46 | SPIN_DELAY(); 47 | goto Retry; 48 | } 49 | if (item->access_count++ == 0) { /* pin page in memory: exclude from LRU list */ 50 | imcs_unlink(pid); 51 | } 52 | if (mode != PM_READ_ONLY) { /* page will be updated */ 53 | if (item->dirty_index == 0) { /* page was not yet modified */ 54 | cache->dirty_pages[cache->n_dirty_pages] = pid; /* include in dirty pages list */ 55 | item->dirty_index = ++cache->n_dirty_pages; 56 | } 57 | } 58 | SpinLockRelease(&cache->mutex); 59 | return IMCS_PAGE_DATA(cache, pid); 60 | } 61 | } 62 | if (cache->free_items_chain != 0) { 63 | pid = cache->free_items_chain; 64 | cache->free_items_chain = cache->items[pid].next; 65 | } else if (cache->n_used_items < imcs_cache_size) { 66 | pid = ++cache->n_used_items; 67 | } else { /* no free items, replace LRU item */ 68 | size_t vh; 69 | int* pp; 70 | pid = cache->items->prev; /* LRU victim */ 71 | if (pid == 0) { /* no free pages */ 72 | imcs_ereport(ERRCODE_OUT_OF_MEMORY, "no available page in cache"); 73 | } 74 | item = &cache->items[pid]; 75 | 76 | /* exclude item from hash table */ 77 | vh = (size_t)(item->offs / imcs_page_size) % imcs_cache_size; 78 | for (pp = &cache->hash_table[vh]; *pp != pid; pp = &cache->items[*pp].collision) { 79 | Assert(*pp != 0); /* item should be present in collision chain */ 80 | } 81 | *pp = item->collision; 82 | 83 | /* exclude item from LRU list */ 84 | imcs_unlink(pid); 85 | 86 | /* save dirty page */ 87 | if (item->dirty_index) { 88 | pg = IMCS_PAGE_DATA(cache, pid); 89 | imcs_file_write(imcs_file, pg, imcs_page_size, item->offs); 90 | cache->dirty_pages[item->dirty_index-1] = cache->dirty_pages[--cache->n_dirty_pages]; /* exclude from dirty list */ 91 | } 92 | } 93 | item = &cache->items[pid]; 94 | if (mode != PM_NEW) { 95 | /* prepare to load page from the disk: mark it as busy to avoid redundant reads */ 96 | pg = IMCS_PAGE_DATA(cache, pid); 97 | item->is_busy = true; 98 | SpinLockRelease(&cache->mutex); /* release mutex during IO */ 99 | imcs_file_read(imcs_file, pg, imcs_page_size, offs); /* read page */ 100 | SpinLockAcquire(&cache->mutex); 101 | } 102 | if (mode != PM_READ_ONLY) { /* include page in dirty list */ 103 | cache->dirty_pages[cache->n_dirty_pages] = pid; 104 | item->dirty_index = ++cache->n_dirty_pages; 105 | } else { 106 | item->dirty_index = 0; 107 | } 108 | item->offs = offs; 109 | item->collision = cache->hash_table[h]; 110 | cache->hash_table[h] = pid; 111 | item->access_count = 1; 112 | item->is_busy = false; 113 | SpinLockRelease(&cache->mutex); 114 | return IMCS_PAGE_DATA(cache, pid); 115 | } 116 | 117 | void imcs_unload_page(imcs_page_t* pg) 118 | { 119 | imcs_disk_cache_t* cache = imcs_disk_cache; 120 | size_t pid = ((char*)pg - cache->data)/imcs_page_size + 1; 121 | imcs_cache_item_t* item = &cache->items[pid]; 122 | Assert(pid-1 < (size_t)imcs_cache_size); 123 | SpinLockAcquire(&cache->mutex); 124 | if (--item->access_count == 0) { /* unpin page */ 125 | imcs_link_after(pg->is_leaf ? cache->lru_internal : 0, pid); 126 | if (!pg->is_leaf && cache->lru_internal == 0) { 127 | cache->lru_internal = pid; 128 | } 129 | } 130 | SpinLockRelease(&cache->mutex); 131 | } 132 | 133 | void imcs_disk_initialize(imcs_disk_cache_t* cache) 134 | { 135 | memset(cache, 0, sizeof(*cache)); 136 | cache->items = (imcs_cache_item_t*)ShmemAlloc((imcs_cache_size+1)*sizeof(imcs_cache_item_t)); 137 | if (cache->items == NULL) { 138 | imcs_ereport(ERRCODE_OUT_OF_MEMORY, "not enough shared memory for disk cache"); 139 | } 140 | cache->data = (char*)ShmemAlloc((size_t)imcs_cache_size*imcs_page_size); 141 | if (cache->data == NULL) { 142 | imcs_ereport(ERRCODE_OUT_OF_MEMORY, "not enough shared memory for disk cache"); 143 | } 144 | cache->hash_table = (int*)ShmemAlloc(imcs_cache_size*sizeof(int)); 145 | if (cache->hash_table == NULL) { 146 | imcs_ereport(ERRCODE_OUT_OF_MEMORY, "not enough shared memory for disk cache"); 147 | } 148 | memset(cache->hash_table, 0, imcs_cache_size*sizeof(int)); 149 | 150 | cache->dirty_pages = (int*)ShmemAlloc(imcs_cache_size*sizeof(int)); 151 | if (cache->dirty_pages == NULL) { 152 | imcs_ereport(ERRCODE_OUT_OF_MEMORY, "not enough shared memory for disk cache"); 153 | } 154 | cache->file_size = imcs_page_size; /* reserve first page to make address not NULL */ 155 | SpinLockInit(&cache->mutex); 156 | imcs_disk_cache = cache; 157 | } 158 | 159 | 160 | void imcs_disk_open(void) 161 | { 162 | imcs_file = imcs_file_open(imcs_file_path); 163 | } 164 | 165 | void imcs_disk_close(void) 166 | { 167 | imcs_file_close(imcs_file); 168 | } 169 | 170 | static int compare_page_offset(void const* p, void const* q) 171 | { 172 | imcs_cache_item_t* p1 = &imcs_disk_cache->items[*(int*)p]; 173 | imcs_cache_item_t* p2 = &imcs_disk_cache->items[*(int*)q]; 174 | return p1->offs < p2->offs ? -1 : p1->offs == p2->offs ? 0 : 1; 175 | } 176 | 177 | void imcs_disk_flush(void) 178 | { 179 | imcs_disk_cache_t* cache = imcs_disk_cache; 180 | int i, n; 181 | SpinLockAcquire(&cache->mutex); 182 | n = cache->n_dirty_pages; 183 | if (n != 0) { 184 | /* sort dirty pages by offset so that them will be written in more or less sequential order */ 185 | qsort(cache->dirty_pages, n, sizeof(int), compare_page_offset); 186 | for (i = 0; i < n; i++) { 187 | int pid = cache->dirty_pages[i]; 188 | imcs_cache_item_t* item = &cache->items[pid]; 189 | imcs_page_t* pg = IMCS_PAGE_DATA(cache, pid); 190 | imcs_file_write(imcs_file, pg, imcs_page_size, item->offs); 191 | item->dirty_index = 0; 192 | } 193 | cache->n_dirty_pages = 0; 194 | } 195 | SpinLockRelease(&cache->mutex); 196 | } 197 | 198 | /* This function is called in context protected by imcs->lock */ 199 | imcs_page_t* imcs_new_page(void) 200 | { 201 | imcs_disk_cache_t* cache = imcs_disk_cache; 202 | uint64 addr = cache->free_pages_chain_head; 203 | if (addr != 0) { /* free page list is not empty */ 204 | if (cache->free_pages_chain_head == cache->free_pages_chain_tail) { 205 | /* free page list is now empty */ 206 | cache->free_pages_chain_head = cache->free_pages_chain_tail = 0; 207 | } else { 208 | if (!imcs_file_read(imcs_file, &cache->free_pages_chain_head, sizeof cache->free_pages_chain_head, addr)) { 209 | imcs_ereport(ERRCODE_IO_ERROR, "Failed to read free page"); 210 | } 211 | Assert(cache->free_pages_chain_tail != 0); 212 | } 213 | } else { 214 | addr = cache->file_size; 215 | cache->file_size += imcs_page_size; 216 | } 217 | cache->n_used_pages += 1; 218 | return (imcs_page_t*)(size_t)addr; 219 | } 220 | 221 | /* This function is called in context protected by imcs->lock. 222 | * "page" is address of page in RAM. 223 | * This function deallocates page, include it in free pages list and exclusde correspondent item from cache 224 | */ 225 | void imcs_free_page(imcs_page_t* pg) 226 | { 227 | imcs_disk_cache_t* cache = imcs_disk_cache; 228 | size_t pid = ((char*)pg - cache->data)/imcs_page_size + 1; 229 | imcs_cache_item_t* item = &cache->items[pid]; 230 | int* pp; 231 | size_t h = (size_t)(item->offs / imcs_page_size) % imcs_cache_size; 232 | 233 | Assert(pid-1 < (size_t)imcs_cache_size); 234 | Assert(item->access_count == 1); /* removed page is pinned */ 235 | 236 | /* remove item from hash table */ 237 | for (pp = &cache->hash_table[h]; *pp != pid; pp = &cache->items[*pp].collision) { 238 | Assert(*pp != 0); /* item should be present in collision chain */ 239 | } 240 | *pp = item->collision; 241 | 242 | /* exclude page from dirty list */ 243 | if (item->dirty_index) { 244 | cache->dirty_pages[item->dirty_index-1] = cache->dirty_pages[--cache->n_dirty_pages]; 245 | } 246 | 247 | /* include item in free items list */ 248 | item->next = cache->free_items_chain; 249 | cache->free_items_chain = pid; 250 | 251 | /* append page to free pages list */ 252 | if (cache->free_pages_chain_tail != 0) { 253 | imcs_file_write(imcs_file, &item->offs, sizeof item->offs, cache->free_pages_chain_tail); 254 | } else { 255 | cache->free_pages_chain_head = item->offs; 256 | } 257 | cache->free_pages_chain_tail = item->offs; 258 | cache->n_used_pages -= 1; 259 | } 260 | 261 | uint64 imcs_used_memory(void) 262 | { 263 | return imcs_disk_cache == NULL ? 0 : imcs_disk_cache->n_used_pages*imcs_page_size; 264 | } 265 | -------------------------------------------------------------------------------- /disk.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Implementation of disk layer for IMCS 3 | */ 4 | #ifndef __DISK_H__ 5 | #define __DISK_H__ 6 | 7 | #include "imcs.h" 8 | #include 9 | 10 | /* 11 | * IMCS implements two-level LRU using L2-list. 12 | * Head of the list corresponds to most recently used internal (not leaf) pages, tail - least recently leaf used. 13 | */ 14 | typedef struct imcs_cache_item_t_ 15 | { 16 | uint64 offs; 17 | int collision; /* hash table collision chain */ 18 | int next; /* L2-list to implement LRU */ 19 | int prev; 20 | int dirty_index; /* for dirty page index of page in dirty_pages + 1, 0 otherwise */ 21 | int access_count; /* page access counter */ 22 | volatile bool is_busy; /* page is currently loaded */ 23 | } imcs_cache_item_t; 24 | 25 | typedef struct 26 | { 27 | imcs_cache_item_t* items; /* imcs_cache_item_t[imcs_cache_size+1], first item is used as head of LRU list */ 28 | char* data; /* char[imcs_cache_size*imcs_page_size] */ 29 | int* dirty_pages; /* int[imcs_cache_size] */ 30 | int* hash_table; /* int[imcs_cache_size] */ 31 | int n_dirty_pages; /* number of used items in array dirty_page */ 32 | int n_used_items; /* number of used items in cache (<= imcs_cache_size), initially 0 */ 33 | int free_items_chain; /* L1 list of free pages (linked by "next" field) */ 34 | int lru_internal; /* index of least recently used internal page: it is used to separate in LRU list leaf pages from internal pages */ 35 | uint64 n_used_pages; 36 | uint64 file_size; /* size of data file */ 37 | uint64 free_pages_chain_head; /* head of L1-list of free pages */ 38 | uint64 free_pages_chain_tail; /* tail of L1-list of free pages */ 39 | slock_t mutex; /* spinlock synchronizing access to the cache */ 40 | } imcs_disk_cache_t; 41 | 42 | #ifdef IMCS_DISK_SUPPORT 43 | 44 | #define IMCS_PAGE_DATA(cache, pid) (imcs_page_t*)(cache->data + ((pid)-1)*imcs_page_size) 45 | 46 | typedef enum { 47 | PM_READ_ONLY, 48 | PM_READ_WRITE, 49 | PM_NEW 50 | } imcs_page_access_mode_t; 51 | 52 | #define IMCS_LOAD_PAGE(pg) pg = imcs_load_page(pg, PM_READ_ONLY) 53 | #define IMCS_LOAD_NEW_PAGE(pg) pg = imcs_load_page(pg, PM_NEW) 54 | #define IMCS_LOAD_PAGE_FOR_UPDATE(pg) pg = imcs_load_page(pg, PM_READ_WRITE) 55 | #define IMCS_UNLOAD_PAGE(pg) imcs_unload_page(pg), pg = 0 56 | 57 | imcs_page_t* imcs_load_page(imcs_page_t* pg, imcs_page_access_mode_t mode); 58 | void imcs_unload_page(imcs_page_t* pg); 59 | void imcs_disk_initialize(imcs_disk_cache_t* cache); 60 | void imcs_disk_open(void); 61 | void imcs_disk_close(void); 62 | void imcs_disk_flush(void); 63 | 64 | #else 65 | 66 | #define IMCS_LOAD_PAGE(pg) 67 | #define IMCS_LOAD_NEW_PAGE(pg) 68 | #define IMCS_LOAD_PAGE_FOR_UPDATE(pg) 69 | #define IMCS_UNLOAD_PAGE(pg) 70 | 71 | #define imcs_disk_initialize(cache) 72 | #define imcs_disk_open() 73 | #define imcs_disk_close() 74 | #define imcs_disk_flush() 75 | 76 | #endif 77 | 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /example.sql: -------------------------------------------------------------------------------- 1 | \timing 2 | 3 | -- Table with quotes. Do not use index to make inserts faster 4 | create table Quote (Symbol char(10), Day date, Open real, High real, Low real, Close real, Volume integer); 5 | 6 | -- Load NYSE data for ten year (can be obtained from http://www.garret.ru/NYSE_2003_2013.csv.gz) 7 | \copy Quote from 'NYSE_2003_2013.csv' with csv header; 8 | -- 2m31.435s 9 | -- with triggers: 6m30.939s 10 | 11 | -- This table should actually contain more information about companies, but for this example we need just symbol name 12 | create table Securities (Symbol char(10)); 13 | 14 | -- It is certainly not efficient way of populating Securities table, usually information about all used symbols is available 15 | insert into Securities select distinct Symbol from Quote; 16 | 17 | -- Generate timeseries functions 18 | select cs_create('Quote', 'Day', 'Symbol'); 19 | 20 | select Quote_load(); 21 | --- Time: 10222.079 ms 22 | 23 | -- We will use this view to perform queries for all quotes for symbols 24 | --create view SecurityQuotes as select * from Quote_get(array(select Symbol from Securities)); 25 | create view SecurityQuotes as select (Quote_get(Symbol)).* from Securities; 26 | 27 | 28 | -- Calculate VWAP (volume-weighted average price) for each symbol 29 | select Symbol,cs_sum(Close*Volume) / cs_sum(Volume) as VWAP from SecurityQuotes; 30 | --- Time: 386.528 ms 31 | 32 | -- Show growth days for symbol ABB during first quoter of 2010 33 | select (Quote_project(abb.*,cs_top_max_pos(Close, 10))).* from Quote_get('ABB', date('01-Jan-2010'), date('31-Mar-2010')) abb; 34 | 35 | select (Quote_project(abb.*,cs_filter_pos(Close>Open*1.01))).* from Quote_get('ABB', date('01-Jan-2010'), date('31-Mar-2010')) abb; 36 | 37 | select cs_count(cs_filter_pos(Close>Open*1.01)) from Quote_get('ABB'); 38 | 39 | -- Now calculate VWAP using standard Postgress aggregates. 40 | select Symbol,sum(Close*Volume)/sum(Volume) as VWAP from Quote group by Symbol; 41 | --- Time: 2184.646 ms 42 | 43 | select Symbol,cs_sum(Close*Volume) / cs_sum(Volume) as VWAP from Quote_get('ABB'); 44 | --- Time: 0.506 ms 45 | 46 | select Symbol,sum(Close*Volume)/sum(Volume) as VWAP from Quote group by Symbol having Symbol='ABB'; 47 | --- Time: 2.818 ms 48 | 49 | select cs_sum(Close) from Quote_concat(array(select Symbol from Securities)); 50 | --- Time: 76.167 ms 51 | 52 | 53 | --- Average True Range (ATR) indicator with 14 days period for last quarter of ABB 54 | select cs_window_atr(cs_maxof(High-Low,0|||cs_maxof(cs_abs((High<<1) - Close), cs_abs((Low<<1) - Close))), 14) << 13 from Quote_get('ABB', date('01-Jan-2010'), date('31-Mar-2010')); 55 | 56 | --- Relative Strength Index (RSI) indicator with 14 days period for last quarter of ABB 57 | select 100-(100/(1+cs_window_ema(cs_maxof(cs_diff(Close), 0), 14)/cs_window_ema(cs_maxof(-cs_diff(Close), 0), 14))) from Quote_get('ABB', date('01-Jan-2010'), date('31-Mar-2010')); 58 | 59 | 60 | --- Now place all quotes in single timeseries (no symbol) 61 | select Quote_drop(); 62 | select cs_create('Quote', 'Day'); 63 | 64 | select Quote_load(); 65 | --- Time: 7658.043 ms 66 | 67 | --- Calculate VWAP for the whole timeseries with ~6 millions elments 68 | select cs_sum(Close*Volume) / cs_sum(Volume) as VWAP from Quote_get(); 69 | --- Time: 7.616 ms 70 | 71 | --- Yet another way of calculating VWAP using cs_wavg 72 | select Volume//Close as VWAP from Quote_get(); 73 | --- Time: 6.501 ms 74 | 75 | --- The same query using standard Postgress aggregates 76 | select sum(Close*Volume)/sum(Volume) as VWAP from Quote; 77 | --- Time: 1078.843 ms 78 | 79 | --- Select top 5 symbols with largest average prices 80 | select cs_project(q, cs_top_max_pos((q).avg, 5)) from (select cs_hash_avg(Close, Symbol) q from Quote_get() offset 0) s; 81 | --- Time: 76.214 ms 82 | 83 | --- The same using standard SQL 84 | select avg(Close) ac,Symbol from Quote group by Symbol order by ac desc limit 5; 85 | --- Time: 1621.042 ms 86 | 87 | --- Find longest periods when NYSE is not working 88 | select cs_map(Day, cs_top_max_pos(cs_diff(Day), 5)),cs_top_max(cs_diff(Day), 5) from Quote_get(); 89 | --- Time: 38.448 ms 90 | 91 | --- Number of unique values of close prices for 10 years 92 | select cs_count(cs_unique(cs_sort(Close))) from Quote_get(); 93 | --- Time: 866.869 ms 94 | 95 | select cs_quantile(Close,5) from Quote_get(); 96 | --- Time: 810.362 ms 97 | 98 | --- Delete all records 99 | select StackOptions_delete(); 100 | select sum(Quote_delete(Symbol)) from Securities; -------------------------------------------------------------------------------- /expected/create.out: -------------------------------------------------------------------------------- 1 | create table Quote (Symbol char(10), Day date, Open real, High real, Low real, Close real, Volume integer); 2 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('01-Nov-2013'), 10.2, 11.0, 10.0, 10.5, 100); 3 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('02-Nov-2013'), 20.2, 20.2, 20.2, 20.2, 200); 4 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('04-Nov-2013'), 30.5, 31.0, 30.0, 30.2, 300); 5 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('05-Nov-2013'), 40.5, 41.0, 40.0, 40.2, 400); 6 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('06-Nov-2013'), 50.2, 51.0, 50.0, 50.5, 500); 7 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('ABB', date('03-Nov-2013'), 60.5, 61.0, 70.0, 60.2, 600); 8 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('ABB', date('06-Nov-2013'), 70.2, 71.0, 70.0, 70.5, 700); 9 | create extension imcs; 10 | select cs_create('Quote', 'Day', 'Symbol'); 11 | cs_create 12 | ----------- 13 | 14 | (1 row) 15 | 16 | select Quote_load(); 17 | quote_load 18 | ------------ 19 | 7 20 | (1 row) 21 | 22 | -- Table with varying size strings which are mapped to integers using dictionary 23 | create table CrashLog( 24 | log_time timestamp, ---when this data is produce,selective 25 | uid bigint, ---user id 26 | country varchar,----USA,Chine,Japan etc...cardinal number is ~ 100,has hot spot 27 | city varchar, ---cardinal number~10000,has hot spot 28 | device varchar, ---iphone4,nokiaX etc, cardinal number is 1000,has hot spot 29 | net varchar, -----wifi,Vodafone 3G... cardinal number is 10,hash hot spot 30 | os varchar); ----iOS6.x,Android 2.3 cardinal number is 100,hash hot spot 31 | insert into CrashLog values ('2014-04-14 11:54', 10000001, 'USA', 'New York', 'iPhone4', 'wifi', 'iOS6.x'); 32 | insert into CrashLog values ('2014-04-14 11:55', 10000002, 'Japan', 'Tokio', 'Sony Xperia Z1', 'Vodafone 3G', 'Android 4.4'); 33 | insert into CrashLog values ('2014-04-14 11:56', 10000003, 'China', 'Beijing', 'iPhone5', 'wifi', 'iOS7.x'); 34 | select cs_create('CrashLog', 'log_time'); 35 | cs_create 36 | ----------- 37 | 38 | (1 row) 39 | 40 | select CrashLog_load(); 41 | crashlog_load 42 | --------------- 43 | 3 44 | (1 row) 45 | 46 | select cs_used_memory(); 47 | cs_used_memory 48 | ---------------- 49 | 77824 50 | (1 row) 51 | 52 | -------------------------------------------------------------------------------- /expected/cumagg.out: -------------------------------------------------------------------------------- 1 | select cs_cum_max(Close) from Quote_get('IBM'); 2 | cs_cum_max 3 | ----------------------------------- 4 | float4:{10.5,20.2,30.2,40.2,50.5} 5 | (1 row) 6 | 7 | select cs_cum_min(Close) from Quote_get('IBM'); 8 | cs_cum_min 9 | ----------------------------------- 10 | float4:{10.5,10.5,10.5,10.5,10.5} 11 | (1 row) 12 | 13 | select cs_cum_sum(Close) from Quote_get('IBM'); 14 | cs_cum_sum 15 | ----------------------------------------------------------------------------------- 16 | float8:{10.5,30.7000007629395,60.9000015258789,101.100002288818,151.600002288818} 17 | (1 row) 18 | 19 | select cs_cum_avg(Close) from Quote_get('IBM'); 20 | cs_cum_avg 21 | ----------------------------------------------------------------------------------- 22 | float8:{10.5,15.3500003814697,20.3000005086263,25.2750005722046,30.3200004577637} 23 | (1 row) 24 | 25 | select cs_cum_prd(Close) from Quote_get('IBM'); 26 | cs_cum_prd 27 | --------------------------------------------------------------------------------- 28 | float8:{10.5,212.100008010864,6405.42040374756,257497.9051176,13003644.2084388} 29 | (1 row) 30 | 31 | select cs_cum_var(Close) from Quote_get('IBM'); 32 | cs_cum_var 33 | -------------------------------------------------------------------------------- 34 | float8:{0,23.5225078201292,64.6866720581052,122.766875371933,200.021595678711} 35 | (1 row) 36 | 37 | select cs_cum_dev(Close) from Quote_get('IBM'); 38 | cs_cum_dev 39 | -------------------------------------------------------------------------------- 40 | float8:{0,4.85000080619882,8.04280250025482,11.0800214517812,14.1428991256641} 41 | (1 row) 42 | 43 | -------------------------------------------------------------------------------- /expected/datetime.out: -------------------------------------------------------------------------------- 1 | select cs_project(q.*) from (select cs_cast(Day, 'timestamp') from Quote_get('ABB')) q; 2 | cs_project 3 | ------------------------------ 4 | ("Sun Nov 03 00:00:00 2013") 5 | ("Wed Nov 06 00:00:00 2013") 6 | (2 rows) 7 | 8 | select cs_year(Day) from Quote_get('ABB'); 9 | cs_year 10 | ------------------ 11 | int4:{2013,2013} 12 | (1 row) 13 | 14 | select cs_month(Day) from Quote_get('ABB'); 15 | cs_month 16 | -------------- 17 | int4:{11,11} 18 | (1 row) 19 | 20 | select cs_mday(Day) from Quote_get('ABB'); 21 | cs_mday 22 | ------------ 23 | int4:{3,6} 24 | (1 row) 25 | 26 | select cs_wday(Day) from Quote_get('ABB'); 27 | cs_wday 28 | ------------ 29 | int4:{0,3} 30 | (1 row) 31 | 32 | select cs_week(Day) from Quote_get('ABB'); 33 | cs_week 34 | ---------------- 35 | int4:{722,722} 36 | (1 row) 37 | 38 | select cs_quarter(Day) from Quote_get('ABB'); 39 | cs_quarter 40 | ------------ 41 | int4:{4,4} 42 | (1 row) 43 | 44 | select cs_year(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 45 | cs_year 46 | ------------------ 47 | int8:{2013,2013} 48 | (1 row) 49 | 50 | select cs_month(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 51 | cs_month 52 | -------------- 53 | int8:{11,11} 54 | (1 row) 55 | 56 | select cs_mday(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 57 | cs_mday 58 | ------------ 59 | int8:{3,6} 60 | (1 row) 61 | 62 | select cs_wday(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 63 | cs_wday 64 | ------------ 65 | int8:{0,3} 66 | (1 row) 67 | 68 | select cs_week(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 69 | cs_week 70 | ---------------- 71 | int8:{722,722} 72 | (1 row) 73 | 74 | select cs_quarter(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 75 | cs_quarter 76 | ------------ 77 | int8:{4,4} 78 | (1 row) 79 | 80 | select cs_hour(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 81 | cs_hour 82 | ------------ 83 | int8:{0,0} 84 | (1 row) 85 | 86 | select cs_minute(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 87 | cs_minute 88 | ------------ 89 | int8:{0,0} 90 | (1 row) 91 | 92 | select cs_second(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 93 | cs_second 94 | ------------ 95 | int8:{0,0} 96 | (1 row) 97 | 98 | select cs_cast(cs_const(date '2013-10-11', 'date'), 'timestamp'); 99 | cs_cast 100 | ------------------------------------------ 101 | timestamp:{Fri Oct 11 00:00:00 2013,...} 102 | (1 row) 103 | 104 | select cs_const(timestamp '2013-10-11 10:00', 'timestamp'); 105 | cs_const 106 | ------------------------------------------ 107 | timestamp:{Fri Oct 11 10:00:00 2013,...} 108 | (1 row) 109 | 110 | select cs_cast(cs_const(timestamp '2013-10-11 10:00', 'timestamp'), 'date'); 111 | cs_cast 112 | ----------------------- 113 | date:{10-11-2013,...} 114 | (1 row) 115 | 116 | -------------------------------------------------------------------------------- /expected/drop.out: -------------------------------------------------------------------------------- 1 | select Quote_delete('IBM', date('02-Nov-2013')); 2 | quote_delete 3 | -------------- 4 | 2 5 | (1 row) 6 | 7 | select Quote_count('IBM'); 8 | quote_count 9 | ------------- 10 | 3 11 | (1 row) 12 | 13 | select Day from Quote_get('IBM'); 14 | day 15 | ----------------------------------------- 16 | date:{11-04-2013,11-05-2013,11-06-2013} 17 | (1 row) 18 | 19 | select Quote_delete('ABB', date('03-Nov-2013'), date('06-Nov-2013')); 20 | quote_delete 21 | -------------- 22 | 2 23 | (1 row) 24 | 25 | select Quote_truncate(); 26 | quote_truncate 27 | ---------------- 28 | 29 | (1 row) 30 | 31 | select Quote_count('IBM'); 32 | quote_count 33 | ------------- 34 | 0 35 | (1 row) 36 | 37 | select Quote_count('ABB'); 38 | quote_count 39 | ------------- 40 | 0 41 | (1 row) 42 | 43 | select * from Quote_get('IBM'); 44 | symbol | day | open | high | low | close | volume 45 | --------+-----+------+------+-----+-------+-------- 46 | (0 rows) 47 | 48 | select * from Quote_get('ABB'); 49 | symbol | day | open | high | low | close | volume 50 | --------+-----+------+------+-----+-------+-------- 51 | (0 rows) 52 | 53 | select CrashLog_delete('2014-04-14 11:54', '2014-04-14 11:56'); 54 | crashlog_delete 55 | ----------------- 56 | 3 57 | (1 row) 58 | 59 | select Quote_drop(); 60 | quote_drop 61 | ------------ 62 | 63 | (1 row) 64 | 65 | select CrashLog_drop(); 66 | crashlog_drop 67 | --------------- 68 | 69 | (1 row) 70 | 71 | -------------------------------------------------------------------------------- /expected/grandagg.out: -------------------------------------------------------------------------------- 1 | select Quote_count('IBM'); 2 | quote_count 3 | ------------- 4 | 5 5 | (1 row) 6 | 7 | select Quote_first('IBM'); 8 | quote_first 9 | ------------- 10 | 11-01-2013 11 | (1 row) 12 | 13 | select Quote_last('IBM'); 14 | quote_last 15 | ------------ 16 | 11-06-2013 17 | (1 row) 18 | 19 | select cs_sum(Volume) from Quote_get('IBM'); 20 | cs_sum 21 | -------- 22 | 1500 23 | (1 row) 24 | 25 | select cs_max(Close) from Quote_get(array['ABB','IBM'], date('03-Nov-2013'), date('05-Nov-2013')); 26 | cs_max 27 | ------------------ 28 | 60.2000007629395 29 | 40.2000007629395 30 | (2 rows) 31 | 32 | select cs_min(Close) from Quote_get(array['ABB','IBM']); 33 | cs_min 34 | ------------------ 35 | 60.2000007629395 36 | 10.5 37 | (2 rows) 38 | 39 | select cs_count(Day) from Quote_get('IBM'); 40 | cs_count 41 | ---------- 42 | 5 43 | (1 row) 44 | 45 | select cs_approxdc(Volume/200) from Quote_get('IBM'); 46 | cs_approxdc 47 | ------------- 48 | 3 49 | (1 row) 50 | 51 | select cs_avg(Volume) from Quote_get('IBM'); 52 | cs_avg 53 | -------- 54 | 300 55 | (1 row) 56 | 57 | select cs_prd(High - Low) from Quote_get('IBM'); 58 | cs_prd 59 | -------- 60 | 0 61 | (1 row) 62 | 63 | select cs_var(Open) from Quote_get('IBM'); 64 | cs_var 65 | ----------------- 66 | 201.22160451355 67 | (1 row) 68 | 69 | select cs_dev(Close) from Quote_get('IBM'); 70 | cs_dev 71 | ------------------ 72 | 14.1428992745525 73 | (1 row) 74 | 75 | select cs_sum('int4:{1,2,3}'); 76 | cs_sum 77 | -------- 78 | 6 79 | (1 row) 80 | 81 | select cs_sum(Close) from Quote_concat(array['ABB','IBM']); 82 | cs_sum 83 | ------------------ 84 | 282.300003051758 85 | (1 row) 86 | 87 | select cs_sum(cs_limit(cs_filter_pos(cs_const(1, 'char')), 0, 3)); 88 | cs_sum 89 | -------- 90 | 6 91 | (1 row) 92 | 93 | select cs_sum(cs_cum_sum(cs_limit(cs_const(1, 'int4'), 0, 3))); 94 | cs_sum 95 | -------- 96 | 10 97 | (1 row) 98 | 99 | select cs_median(Close) from Quote_get(array['ABB','IBM']); 100 | cs_median 101 | ------------------ 102 | 65.3499984741211 103 | 30.2000007629395 104 | (2 rows) 105 | 106 | select cs_all('int2:{2,3,6}'); 107 | cs_all 108 | -------- 109 | 2 110 | (1 row) 111 | 112 | select cs_any('char:{2,3,6}'); 113 | cs_any 114 | -------- 115 | 7 116 | (1 row) 117 | 118 | -------------------------------------------------------------------------------- /expected/gridagg.out: -------------------------------------------------------------------------------- 1 | select cs_grid_max(Close,2) from Quote_get('IBM'); 2 | cs_grid_max 3 | ------------------------- 4 | float4:{20.2,40.2,50.5} 5 | (1 row) 6 | 7 | select cs_grid_min(Close,2) from Quote_get('IBM'); 8 | cs_grid_min 9 | ------------------------- 10 | float4:{10.5,30.2,50.5} 11 | (1 row) 12 | 13 | select cs_grid_sum(Close,2) from Quote_get('IBM'); 14 | cs_grid_sum 15 | ------------------------------------------------- 16 | float8:{30.7000007629395,70.4000015258789,50.5} 17 | (1 row) 18 | 19 | select cs_grid_avg(Close,2) from Quote_get('IBM'); 20 | cs_grid_avg 21 | ------------------------------------------------- 22 | float8:{15.3500003814697,35.2000007629395,50.5} 23 | (1 row) 24 | 25 | select cs_grid_var(Close,2) from Quote_get('IBM'); 26 | cs_grid_var 27 | -------------------------------- 28 | float8:{23.5225037002565,25,0} 29 | (1 row) 30 | 31 | select cs_grid_dev(Close,2) from Quote_get('IBM'); 32 | cs_grid_dev 33 | ------------------------------- 34 | float8:{4.85000038146973,5,0} 35 | (1 row) 36 | 37 | -------------------------------------------------------------------------------- /expected/groupbyagg.out: -------------------------------------------------------------------------------- 1 | select cs_group_max(Close,Day/3) from Quote_get('IBM'); 2 | cs_group_max 3 | ------------------------- 4 | float4:{20.2,40.2,50.5} 5 | (1 row) 6 | 7 | select cs_group_min(Close,Day/3) from Quote_get('IBM'); 8 | cs_group_min 9 | ------------------------- 10 | float4:{10.5,30.2,50.5} 11 | (1 row) 12 | 13 | select cs_group_sum(Close,Day/3) from Quote_get('IBM'); 14 | cs_group_sum 15 | ------------------------------------------------- 16 | float8:{30.7000007629395,70.4000015258789,50.5} 17 | (1 row) 18 | 19 | select cs_group_avg(Close,Day/3) from Quote_get('IBM'); 20 | cs_group_avg 21 | ------------------------------------------------- 22 | float8:{15.3500003814697,35.2000007629395,50.5} 23 | (1 row) 24 | 25 | select cs_group_var(Close,Day/3) from Quote_get('IBM'); 26 | cs_group_var 27 | -------------------------------- 28 | float8:{23.5225037002565,25,0} 29 | (1 row) 30 | 31 | select cs_group_dev(Close,Day/3) from Quote_get('IBM'); 32 | cs_group_dev 33 | ------------------------------- 34 | float8:{4.85000038146973,5,0} 35 | (1 row) 36 | 37 | select cs_group_first(Close,Day/3) from Quote_get('IBM'); 38 | cs_group_first 39 | ------------------------- 40 | float4:{10.5,30.2,50.5} 41 | (1 row) 42 | 43 | select cs_group_last(Close,Day/3) from Quote_get('IBM'); 44 | cs_group_last 45 | ------------------------- 46 | float4:{20.2,40.2,50.5} 47 | (1 row) 48 | 49 | select cs_group_all('int8:{3,1,6,7,0,3,6,5,2,3,7}','int4:{1,1,1,2,2,3,3,4,5,5,5}'); 50 | cs_group_all 51 | ------------------ 52 | int8:{0,0,2,5,2} 53 | (1 row) 54 | 55 | select cs_group_any('int2:{3,1,6,7,0,3,6,5,2,3,7}','char:{1,1,1,2,2,3,3,4,5,5,5}'); 56 | cs_group_any 57 | ------------------ 58 | int2:{7,7,7,5,7} 59 | (1 row) 60 | 61 | select cs_win_group_max(Close,cs_week(Day)) from Quote_get('IBM'); 62 | cs_win_group_max 63 | ----------------------------------- 64 | float4:{20.2,20.2,50.5,50.5,50.5} 65 | (1 row) 66 | 67 | select cs_win_group_min(Close,cs_week(Day)) from Quote_get('IBM'); 68 | cs_win_group_min 69 | ----------------------------------- 70 | float4:{10.5,10.5,30.2,30.2,30.2} 71 | (1 row) 72 | 73 | select cs_win_group_sum(Close,cs_week(Day)) from Quote_get('IBM'); 74 | cs_win_group_sum 75 | ----------------------------------------------------------------------------------------------- 76 | float8:{30.7000007629395,30.7000007629395,120.900001525879,120.900001525879,120.900001525879} 77 | (1 row) 78 | 79 | select cs_win_group_avg(Close,cs_week(Day)) from Quote_get('IBM'); 80 | cs_win_group_avg 81 | ----------------------------------------------------------------------------------------------- 82 | float8:{15.3500003814697,15.3500003814697,40.3000005086263,40.3000005086263,40.3000005086263} 83 | (1 row) 84 | 85 | select cs_win_group_var(Close,cs_week(Day)) from Quote_get('IBM'); 86 | cs_win_group_var 87 | ----------------------------------------------------------------------------------------------- 88 | float8:{23.5225037002565,23.5225037002565,68.6866614786786,68.6866614786786,68.6866614786786} 89 | (1 row) 90 | 91 | select cs_win_group_dev(Close,cs_week(Day)) from Quote_get('IBM'); 92 | cs_win_group_dev 93 | ----------------------------------------------------------------------------------------------- 94 | float8:{4.85000038146973,4.85000038146973,8.28774163923313,8.28774163923313,8.28774163923313} 95 | (1 row) 96 | 97 | select cs_win_group_first(Close,cs_week(Day)) from Quote_get('IBM'); 98 | cs_win_group_first 99 | ----------------------------------- 100 | float4:{10.5,10.5,30.2,30.2,30.2} 101 | (1 row) 102 | 103 | select cs_win_group_last(Close,cs_week(Day)) from Quote_get('IBM'); 104 | cs_win_group_last 105 | ----------------------------------- 106 | float4:{20.2,20.2,50.5,50.5,50.5} 107 | (1 row) 108 | 109 | select cs_win_group_sum('int4:{1,2,3,4,5,6,7,8,9,10}','int4:{1,1,1,2,2,3,3,3,3,4}'); 110 | cs_win_group_sum 111 | --------------------------------- 112 | int8:{6,6,6,9,9,30,30,30,30,10} 113 | (1 row) 114 | 115 | -------------------------------------------------------------------------------- /expected/hashagg.out: -------------------------------------------------------------------------------- 1 | select cs_hash_max(Close,Day % 2) from Quote_get('IBM'); 2 | cs_hash_max 3 | ------------------------------------- 4 | ("float4:{50.5,40.2}","int4:{0,1}") 5 | (1 row) 6 | 7 | select cs_hash_min(Close,Day % 2) from Quote_get('IBM'); 8 | cs_hash_min 9 | ------------------------------------- 10 | ("float4:{20.2,10.5}","int4:{0,1}") 11 | (1 row) 12 | 13 | select cs_hash_sum(Close,Day % 2) from Quote_get('IBM'); 14 | cs_hash_sum 15 | ------------------------------------------------------------- 16 | ("float8:{100.900001525879,50.7000007629395}","int4:{0,1}") 17 | (1 row) 18 | 19 | select cs_hash_avg(Close,Day % 2) from Quote_get('IBM'); 20 | cs_hash_avg 21 | ------------------------------------------------------------- 22 | ("float8:{33.6333338419596,25.3500003814697}","int4:{0,1}") 23 | (1 row) 24 | 25 | select (q.p).agg_val,cs_cut((q.p).group_by,'i4i4') from (select cs_project_agg(cs_hash_sum(Close,(Day % 2)||(Volume%10))) p from Quote_get('IBM')) q; 26 | agg_val | cs_cut 27 | ------------------+-------- 28 | 50.7000007629395 | (1,0) 29 | 100.900001525879 | (0,0) 30 | (2 rows) 31 | 32 | select p.agg_val,cs_cut(p.group_by,'i4i4') from (select (cs_project_agg(cs_hash_sum(Close,(Day % 2)||(Volume%10)))).* from Quote_get('IBM')) p; 33 | agg_val | cs_cut 34 | ------------------+-------- 35 | 50.7000007629395 | (1,0) 36 | 100.900001525879 | (0,0) 37 | (2 rows) 38 | 39 | create type PairOfInt as (first integer, second integer); 40 | select p.agg_val,cs_as(p.group_by,'PairOfInt') from (select (cs_project_agg(cs_hash_sum(Close,(Day % 2)||(Volume%10)))).* from Quote_get('IBM')) p; 41 | agg_val | cs_as 42 | ------------------+------- 43 | 50.7000007629395 | (1,0) 44 | 100.900001525879 | (0,0) 45 | (2 rows) 46 | 47 | select cs_hash_count(cs_floor((High-Low)*10)) from Quote_get('IBM'); 48 | cs_hash_count 49 | -------------------------------- 50 | ("int8:{4,1}","float8:{10,0}") 51 | (1 row) 52 | 53 | select cs_hash_dup_count(cs_ceil((High-Low)*10), Day%3) from Quote_get('IBM'); 54 | cs_hash_dup_count 55 | --------------------------------- 56 | ("int8:{1,1,2}","int4:{1,0,2}") 57 | (1 row) 58 | 59 | select cs_hash_all('int4:{3,1,6,7,0,3,6,5,2,3,7}','int8:{1,1,1,2,2,3,3,4,5,5,5}'); 60 | cs_hash_all 61 | ----------------------------------------- 62 | ("int4:{0,2,5,0,2}","int8:{1,5,4,2,3}") 63 | (1 row) 64 | 65 | select cs_hash_any('char:{3,1,6,7,0,3,6,5,2,3,7}','int2:{1,1,1,2,2,3,3,4,5,5,5}'); 66 | cs_hash_any 67 | ----------------------------------------- 68 | ("char:{7,7,7,5,7}","int2:{2,1,5,4,3}") 69 | (1 row) 70 | 71 | select cs_code2str(group_by, 1) as country, cs_code2str(group_by, 2) as device, agg_val as counter from (select (cs_project_agg(cs_hash_count(country||device))).* from CrashLog_get()) s; 72 | country | device | counter 73 | ---------+----------------+--------- 74 | Japan | Sony Xperia Z1 | 1 75 | USA | iPhone4 | 1 76 | China | iPhone5 | 1 77 | (3 rows) 78 | 79 | -------------------------------------------------------------------------------- /expected/math.out: -------------------------------------------------------------------------------- 1 | select cs_sqrt((Open - Close) ^ 2.0) from Quote_get('IBM'); 2 | cs_sqrt 3 | ------------------------------------------------------------------------------------ 4 | float8:{0.300000190734863,0,0.299999237060547,0.299999237060547,0.299999237060547} 5 | (1 row) 6 | 7 | select cs_sin(Open)*cs_sin(Open) + cs_cos(Open)*cs_cos(Open) from Quote_get('IBM'); 8 | ?column? 9 | -------------------- 10 | float8:{1,1,1,1,1} 11 | (1 row) 12 | 13 | select cs_atan(cs_tan(Close)) from Quote_get('IBM'); 14 | cs_atan 15 | --------------------------------------------------------------------------------------------------- 16 | float8:{1.07522203923062,1.35044484140069,-1.21592577295848,-0.640703733727859,0.234517542563308} 17 | (1 row) 18 | 19 | select cs_asin(cs_sin(Close)) from Quote_get('IBM'); 20 | cs_asin 21 | --------------------------------------------------------------------------------------------------- 22 | float8:{-1.07522203923062,1.35044484140069,-1.21592577295848,0.640703733727859,0.234517542563308} 23 | (1 row) 24 | 25 | select cs_acos(cs_cos(Close)) from Quote_get('IBM'); 26 | cs_acos 27 | ------------------------------------------------------------------------------------------------ 28 | float8:{2.06637061435917,1.35044484140069,1.21592577295848,2.50088891986193,0.234517542563308} 29 | (1 row) 30 | 31 | select cs_log(cs_exp(Close/Open)) - Open/Close from Quote_get('IBM'); 32 | ?column? 33 | ------------------------------------------------------------------------------------------ 34 | float8:{0.0579832196235657,0,-0.0197697281837463,-0.0148699879646301,0.0119166374206543} 35 | (1 row) 36 | 37 | select cs_isnan(cs_parse('{-1,0,1}','float8')/0.0); 38 | cs_isnan 39 | -------------- 40 | char:{0,1,0} 41 | (1 row) 42 | 43 | -------------------------------------------------------------------------------- /expected/operators.out: -------------------------------------------------------------------------------- 1 | select Open+Close from Quote_get('IBM'); 2 | ?column? 3 | ------------------------------------ 4 | float4:{20.7,40.4,60.7,80.7,100.7} 5 | (1 row) 6 | 7 | select Open-Close from Quote_get('IBM'); 8 | ?column? 9 | --------------------------------------------- 10 | float4:{-0.3,0,0.299999,0.299999,-0.299999} 11 | (1 row) 12 | 13 | select Open*Close from Quote_get('IBM'); 14 | ?column? 15 | ------------------------------------------- 16 | float4:{107.1,408.04,921.1,1628.1,2535.1} 17 | (1 row) 18 | 19 | select Open/Close from Quote_get('IBM'); 20 | ?column? 21 | ---------------------------------------------- 22 | float4:{0.971429,1,1.00993,1.00746,0.994059} 23 | (1 row) 24 | 25 | select Open%Close from Quote_get('IBM'); 26 | ?column? 27 | ---------------------------------------- 28 | float4:{10.2,0,0.299999,0.299999,50.2} 29 | (1 row) 30 | 31 | select Open^Close from Quote_get('IBM'); 32 | ?column? 33 | -------------------------------------------------------------------------------------------------------------- 34 | float8:{38931552097.3912,2.33398999511658e+26,6.6966296391943e+44,4.16574958399165e+64,7.68312765681365e+85} 35 | (1 row) 36 | 37 | select Close+*Volume from Quote_get('IBM'); 38 | ?column? 39 | ------------------ 40 | 55480.0006866455 41 | (1 row) 42 | 43 | select Volume//Close from Quote_get('IBM'); 44 | ?column? 45 | ------------------ 46 | 36.9866668294271 47 | (1 row) 48 | 49 | select Open~Close from Quote_get('IBM'); 50 | ?column? 51 | ------------------ 52 | 0.99982502975582 53 | (1 row) 54 | 55 | select (Open>Close)?Day from Quote_get('IBM'); 56 | ?column? 57 | ------------------------------ 58 | date:{11-04-2013,11-05-2013} 59 | (1 row) 60 | 61 | select ?(Open>Close) from Quote_get('IBM'); 62 | ?column? 63 | ------------ 64 | int8:{2,3} 65 | (1 row) 66 | 67 | select -Open from Quote_get('IBM'); 68 | ?column? 69 | ---------------------------------------- 70 | float4:{-10.2,-20.2,-30.5,-40.5,-50.2} 71 | (1 row) 72 | 73 | select Open>Close from Quote_get('IBM'); 74 | ?column? 75 | ------------------ 76 | char:{0,0,1,1,0} 77 | (1 row) 78 | 79 | select Open>=Close from Quote_get('IBM'); 80 | ?column? 81 | ------------------ 82 | char:{0,1,1,1,0} 83 | (1 row) 84 | 85 | select OpenClose from Quote_get('IBM'); 104 | ?column? 105 | ------------------ 106 | char:{1,0,1,1,1} 107 | (1 row) 108 | 109 | select (Open>=Close) & (High<>Low) from Quote_get('IBM'); 110 | ?column? 111 | ------------------ 112 | char:{0,0,1,1,0} 113 | (1 row) 114 | 115 | select (Open>Close) | (High=Low) from Quote_get('IBM'); 116 | ?column? 117 | ------------------ 118 | char:{0,1,1,1,0} 119 | (1 row) 120 | 121 | select (Open=Close) # (High=Low) from Quote_get('IBM'); 122 | ?column? 123 | ------------------ 124 | char:{0,0,0,0,0} 125 | (1 row) 126 | 127 | select !(Open=Close) from Quote_get('IBM'); 128 | ?column? 129 | ------------------ 130 | char:{1,0,1,1,1} 131 | (1 row) 132 | 133 | select ~Volume from Quote_get('IBM'); 134 | ?column? 135 | --------------------------------- 136 | int4:{-101,-201,-301,-401,-501} 137 | (1 row) 138 | 139 | select Volume & 1 from Quote_get('IBM'); 140 | ?column? 141 | ------------------ 142 | int4:{0,0,0,0,0} 143 | (1 row) 144 | 145 | select Volume | 1 from Quote_get('IBM'); 146 | ?column? 147 | ---------------------------- 148 | int4:{101,201,301,401,501} 149 | (1 row) 150 | 151 | select Volume # -1 from Quote_get('IBM'); 152 | ?column? 153 | --------------------------------- 154 | int4:{-101,-201,-301,-401,-501} 155 | (1 row) 156 | 157 | select @(Open-Close) from Quote_get('IBM'); 158 | ?column? 159 | ------------------------------------------- 160 | float4:{0.3,0,0.299999,0.299999,0.299999} 161 | (1 row) 162 | 163 | select Day,Open||Close from Quote_get('IBM', date('01-Nov-2013'), date('03-Nov-2013')); 164 | day | ?column? 165 | ------------------------------+------------------- 166 | date:{11-01-2013,11-02-2013} | bpchar8:{33#A,AA} 167 | (1 row) 168 | 169 | select cs_maxof(Open,Close) from Quote_get('IBM'); 170 | cs_maxof 171 | ----------------------------------- 172 | float4:{10.5,20.2,30.5,40.5,50.5} 173 | (1 row) 174 | 175 | select cs_minof(Open,Close) from Quote_get('IBM'); 176 | cs_minof 177 | ----------------------------------- 178 | float4:{10.2,20.2,30.2,40.2,50.2} 179 | (1 row) 180 | 181 | select cs_norm(Close) from Quote_get('IBM'); 182 | cs_norm 183 | --------------------------------------------------------------------------------------------------- 184 | float8:{0.140354513651518,0.270015360270729,0.403686325653127,0.537357291035525,0.67503837518111} 185 | (1 row) 186 | 187 | select Close*'{2.0,2.1,2.2}'::text from Quote_get('IBM', '1-Nov-2013', '4-Nov-2013'); 188 | ?column? 189 | ------------------------- 190 | float4:{21,42.42,66.44} 191 | (1 row) 192 | 193 | select Close*'float4:{2.0,2.1,2.2}' from Quote_get('IBM', '1-Nov-2013', '4-Nov-2013'); 194 | ?column? 195 | ------------------------- 196 | float4:{21,42.42,66.44} 197 | (1 row) 198 | 199 | select Day=date('1-Nov-2013') from Quote_get('IBM'); 200 | ?column? 201 | ------------------ 202 | char:{1,0,0,0,0} 203 | (1 row) 204 | 205 | select Volume*2 from Quote_get('ABB'); 206 | ?column? 207 | ------------------ 208 | int4:{1200,1400} 209 | (1 row) 210 | 211 | select cs_parse('{1,2,3,4}','int4')+cs_const(3.14,'float4'); 212 | ?column? 213 | ------------------------------ 214 | float4:{4.14,5.14,6.14,7.14} 215 | (1 row) 216 | 217 | select cs_parse('{1,2,3,4}','int4')+10; 218 | ?column? 219 | -------------------- 220 | int4:{11,12,13,14} 221 | (1 row) 222 | 223 | select cs_parse('{1,2,3,4}', 'int4')+'{10,20,30,40}'::text; 224 | ?column? 225 | -------------------- 226 | int4:{11,22,33,44} 227 | (1 row) 228 | 229 | select cs_parse('{11-Nov-2013,12-Nov-2013,30-Nov-2013}', 'date'); 230 | cs_parse 231 | ----------------------------------------- 232 | date:{11-11-2013,11-12-2013,11-30-2013} 233 | (1 row) 234 | 235 | select cs_concat('int4:{1,2,3}','int4:{4,5,6}'); 236 | cs_concat 237 | -------------------- 238 | int4:{1,2,3,4,5,6} 239 | (1 row) 240 | 241 | select cs_parse('{1,2,3,4,5}', 'int4') << 1; 242 | ?column? 243 | ---------------- 244 | int4:{2,3,4,5} 245 | (1 row) 246 | 247 | select cs_parse('{1,2,3,4,5}', 'int4') >> 2; 248 | ?column? 249 | -------------- 250 | int4:{1,2,3} 251 | (1 row) 252 | 253 | select cs_parse('{1,2,3,4,5}', 'int4') << 10; 254 | ?column? 255 | ---------- 256 | int4:{} 257 | (1 row) 258 | 259 | select cs_parse('{1,2,3,4,5}', 'int4') >> 10; 260 | ?column? 261 | ---------- 262 | int4:{} 263 | (1 row) 264 | 265 | select cs_like('bpchar8:{Abc,aaBcc,aabbcc,abcba,aabbabcc,aabac,bca}', '%abc%'); 266 | cs_like 267 | ---------------------- 268 | char:{0,0,0,1,1,0,0} 269 | (1 row) 270 | 271 | select cs_ilike('bpchar8:{Abc,aaBcc,aabbcc,abcba,aabbabcc,aabac,bca}', '%abc%'); 272 | cs_ilike 273 | ---------------------- 274 | char:{1,1,0,1,1,0,0} 275 | (1 row) 276 | 277 | select cs_parse('{Abc,aaBcc,aabbcc,abcba,aabbabcc,aabac,bca}', 'bpchar', 8) ~~ '%abc%'; 278 | ?column? 279 | ---------------------- 280 | char:{1,1,0,1,1,0,0} 281 | (1 row) 282 | 283 | select * from CrashLog_get(); 284 | log_time | uid | country | city | device | net | os 285 | ----------------------------------------------------------------------------------------+-----------------------------------+---------------------------+----------------------------------+------------------------------------------+---------------------------------+------------------------------------- 286 | timestamp:{Mon Apr 14 11:54:00 2014,Mon Apr 14 11:55:00 2014,Mon Apr 14 11:56:00 2014} | int8:{10000001,10000002,10000003} | varchar:{USA,Japan,China} | varchar:{New York,Tokio,Beijing} | varchar:{iPhone4,Sony Xperia Z1,iPhone5} | varchar:{wifi,Vodafone 3G,wifi} | varchar:{iOS6.x,Android 4.4,iOS7.x} 287 | (1 row) 288 | 289 | select cs_project(c.*, cs_filter_pos(country = cs_str2code('USA'))) from CrashLog_get() c; 290 | cs_project 291 | -------------------------------------------------------------------------- 292 | ("Mon Apr 14 11:54:00 2014",10000001,USA,"New York",iPhone4,wifi,iOS6.x) 293 | (1 row) 294 | 295 | select cs_project(c.*, cs_filter_pos(cs_ilike(device, 'iphone%'))) from CrashLog_get() c; 296 | cs_project 297 | -------------------------------------------------------------------------- 298 | ("Mon Apr 14 11:54:00 2014",10000001,USA,"New York",iPhone4,wifi,iOS6.x) 299 | ("Mon Apr 14 11:56:00 2014",10000003,China,Beijing,iPhone5,wifi,iOS7.x) 300 | (2 rows) 301 | 302 | select cs_dictionary_size(); 303 | cs_dictionary_size 304 | -------------------- 305 | 14 306 | (1 row) 307 | 308 | -------------------------------------------------------------------------------- /expected/scalarop.out: -------------------------------------------------------------------------------- 1 | select cs_wsum(Volume,Close) from Quote_get('IBM'); 2 | cs_wsum 3 | ------------------ 4 | 55480.0006866455 5 | (1 row) 6 | 7 | select cs_wavg(Volume,Close) from Quote_get('IBM'); 8 | cs_wavg 9 | ------------------ 10 | 36.9866668294271 11 | (1 row) 12 | 13 | select cs_corr(High,Low) from Quote_get('IBM'); 14 | cs_corr 15 | ------------------- 16 | 0.999654253859527 17 | (1 row) 18 | 19 | select cs_cov(High,Low) from Quote_get('IBM'); 20 | cs_cov 21 | --------------- 22 | 201.174396875 23 | (1 row) 24 | 25 | -------------------------------------------------------------------------------- /expected/sort.out: -------------------------------------------------------------------------------- 1 | select cs_top_max(Close, 3) from Quote_get('IBM'); 2 | cs_top_max 3 | ------------------------- 4 | float4:{50.5,40.2,30.2} 5 | (1 row) 6 | 7 | select cs_top_min(Close, 3) from Quote_get('IBM'); 8 | cs_top_min 9 | ------------------------- 10 | float4:{10.5,20.2,30.2} 11 | (1 row) 12 | 13 | select cs_top_max_pos(Close, 3) from Quote_get('IBM'); 14 | cs_top_max_pos 15 | ---------------- 16 | int8:{4,3,2} 17 | (1 row) 18 | 19 | select cs_top_min_pos(Close, 3) from Quote_get('IBM'); 20 | cs_top_min_pos 21 | ---------------- 22 | int8:{0,1,2} 23 | (1 row) 24 | 25 | select Quote_project(q.*, cs_top_max_pos(Close, 3)) from Quote_get('IBM') q; 26 | quote_project 27 | -------------------------------------- 28 | (IBM,11-06-2013,50.2,51,50,50.5,500) 29 | (IBM,11-05-2013,40.5,41,40,40.2,400) 30 | (IBM,11-04-2013,30.5,31,30,30.2,300) 31 | (3 rows) 32 | 33 | select cs_sort(Close) from Quote_get('IBM'); 34 | cs_sort 35 | ----------------------------------- 36 | float4:{10.5,20.2,30.2,40.2,50.5} 37 | (1 row) 38 | 39 | select cs_sort(Close, 'desc') from Quote_get('IBM'); 40 | cs_sort 41 | ----------------------------------- 42 | float4:{50.5,40.2,30.2,20.2,10.5} 43 | (1 row) 44 | 45 | select Quote_project(q.*, cs_sort_pos(Close)) from Quote_get('IBM') q; 46 | quote_project 47 | ------------------------------------------ 48 | (IBM,11-01-2013,10.2,11,10,10.5,100) 49 | (IBM,11-02-2013,20.2,20.2,20.2,20.2,200) 50 | (IBM,11-04-2013,30.5,31,30,30.2,300) 51 | (IBM,11-05-2013,40.5,41,40,40.2,400) 52 | (IBM,11-06-2013,50.2,51,50,50.5,500) 53 | (5 rows) 54 | 55 | select Quote_project(q.*, cs_sort_pos(Close, 'desc')) from Quote_get('IBM') q; 56 | quote_project 57 | ------------------------------------------ 58 | (IBM,11-06-2013,50.2,51,50,50.5,500) 59 | (IBM,11-05-2013,40.5,41,40,40.2,400) 60 | (IBM,11-04-2013,30.5,31,30,30.2,300) 61 | (IBM,11-02-2013,20.2,20.2,20.2,20.2,200) 62 | (IBM,11-01-2013,10.2,11,10,10.5,100) 63 | (5 rows) 64 | 65 | select cs_rank(Close) from Quote_get('IBM'); 66 | cs_rank 67 | ------------------ 68 | int8:{1,2,3,4,5} 69 | (1 row) 70 | 71 | select cs_rank('float4:{1.1,0.1,2.2,0.2,0.1}'); 72 | cs_rank 73 | ------------------ 74 | int8:{4,1,5,3,1} 75 | (1 row) 76 | 77 | select cs_dense_rank('float4:{1.1,0.1,2.2,0.2,0.1}'); 78 | cs_dense_rank 79 | ------------------ 80 | int8:{3,1,4,2,1} 81 | (1 row) 82 | 83 | select cs_rank('float4:{1.1,0.2,2.2,0.2,0.1}', 'desc'); 84 | cs_rank 85 | ------------------ 86 | int8:{2,3,1,3,5} 87 | (1 row) 88 | 89 | select cs_dense_rank('float4:{1.1,0.2,2.2,0.2,0.1}', 'desc'); 90 | cs_dense_rank 91 | ------------------ 92 | int8:{2,3,1,3,4} 93 | (1 row) 94 | 95 | select cs_quantile(Close, 2) from Quote_get('IBM'); 96 | cs_quantile 97 | ------------------------- 98 | float4:{10.5,30.2,50.5} 99 | (1 row) 100 | 101 | select cs_quantile('float4:{10,3,0,3,4,5,9,11,7,3,3}', 2); 102 | cs_quantile 103 | ----------------- 104 | float4:{0,4,11} 105 | (1 row) 106 | 107 | -------------------------------------------------------------------------------- /expected/span.out: -------------------------------------------------------------------------------- 1 | select Close from Quote_get('IBM', '2-Nov-2013', '5-Nov-2013'); 2 | close 3 | ------------------------- 4 | float4:{20.2,30.2,40.2} 5 | (1 row) 6 | 7 | select Close from Quote_get('IBM', '2-Nov-2013'); 8 | close 9 | ------------------------------ 10 | float4:{20.2,30.2,40.2,50.5} 11 | (1 row) 12 | 13 | select Close from Quote_get('IBM', from_ts:='2-Nov-2013'); 14 | close 15 | ------------------------------ 16 | float4:{20.2,30.2,40.2,50.5} 17 | (1 row) 18 | 19 | select Close from Quote_get('IBM', till_ts:='5-Nov-2013'); 20 | close 21 | ------------------------------ 22 | float4:{10.5,20.2,30.2,40.2} 23 | (1 row) 24 | 25 | select Close from Quote_get('IBM'); 26 | close 27 | ----------------------------------- 28 | float4:{10.5,20.2,30.2,40.2,50.5} 29 | (1 row) 30 | 31 | select Close from Quote_span('IBM', 1, 3); 32 | close 33 | ------------------------- 34 | float4:{20.2,30.2,40.2} 35 | (1 row) 36 | 37 | select Close from Quote_span('IBM', 1); 38 | close 39 | ------------------------------ 40 | float4:{20.2,30.2,40.2,50.5} 41 | (1 row) 42 | 43 | select Close from Quote_span('IBM', from_pos:=1); 44 | close 45 | ------------------------------ 46 | float4:{20.2,30.2,40.2,50.5} 47 | (1 row) 48 | 49 | select Close from Quote_span('IBM', till_pos:=3); 50 | close 51 | ------------------------------ 52 | float4:{10.5,20.2,30.2,40.2} 53 | (1 row) 54 | 55 | select Close from Quote_span('IBM'); 56 | close 57 | ----------------------------------- 58 | float4:{10.5,20.2,30.2,40.2,50.5} 59 | (1 row) 60 | 61 | select Close from Quote_get(array['ABB','IBM'], '2-Nov-2013', '5-Nov-2013'); 62 | close 63 | ------------------------- 64 | float4:{60.2} 65 | float4:{20.2,30.2,40.2} 66 | (2 rows) 67 | 68 | select Close from Quote_get(array['ABB','IBM'], '2-Nov-2013'); 69 | close 70 | ------------------------------ 71 | float4:{60.2,70.5} 72 | float4:{20.2,30.2,40.2,50.5} 73 | (2 rows) 74 | 75 | select Close from Quote_get(array['ABB','IBM'], from_ts:='2-Nov-2013'); 76 | close 77 | ------------------------------ 78 | float4:{60.2,70.5} 79 | float4:{20.2,30.2,40.2,50.5} 80 | (2 rows) 81 | 82 | select Close from Quote_get(array['ABB','IBM'], till_ts:='5-Nov-2013'); 83 | close 84 | ------------------------------ 85 | float4:{60.2} 86 | float4:{10.5,20.2,30.2,40.2} 87 | (2 rows) 88 | 89 | select Close from Quote_get(array['ABB','IBM']); 90 | close 91 | ----------------------------------- 92 | float4:{60.2,70.5} 93 | float4:{10.5,20.2,30.2,40.2,50.5} 94 | (2 rows) 95 | 96 | select Close from Quote_span(array['ABB','IBM'], 1, 3); 97 | close 98 | ------------------------- 99 | float4:{70.5} 100 | float4:{20.2,30.2,40.2} 101 | (2 rows) 102 | 103 | select Close from Quote_span(array['ABB','IBM'], 1); 104 | close 105 | ------------------------------ 106 | float4:{70.5} 107 | float4:{20.2,30.2,40.2,50.5} 108 | (2 rows) 109 | 110 | select Close from Quote_span(array['ABB','IBM'], from_pos:=1); 111 | close 112 | ------------------------------ 113 | float4:{70.5} 114 | float4:{20.2,30.2,40.2,50.5} 115 | (2 rows) 116 | 117 | select Close from Quote_span(array['ABB','IBM'], till_pos:=3); 118 | close 119 | ------------------------------ 120 | float4:{60.2,70.5} 121 | float4:{10.5,20.2,30.2,40.2} 122 | (2 rows) 123 | 124 | select Close from Quote_span(array['ABB','IBM']); 125 | close 126 | ----------------------------------- 127 | float4:{60.2,70.5} 128 | float4:{10.5,20.2,30.2,40.2,50.5} 129 | (2 rows) 130 | 131 | -------------------------------------------------------------------------------- /expected/spec.out: -------------------------------------------------------------------------------- 1 | select cs_histogram(Close, 0, 100, 10) from Quote_get('IBM'); 2 | cs_histogram 3 | ---------------------------- 4 | int8:{0,1,1,1,1,1,0,0,0,0} 5 | (1 row) 6 | 7 | select cs_cross(Open-Close, 0) from Quote_get('IBM'); 8 | cs_cross 9 | ------------ 10 | int8:{1,4} 11 | (1 row) 12 | 13 | select cs_extrema(cs_parse('{1,2,3,2,1,0,0,1,1,2,4,0}','int4'), 0); 14 | cs_extrema 15 | --------------- 16 | int8:{2,6,10} 17 | (1 row) 18 | 19 | select cs_extrema(cs_parse('{1,2,3,2,1,0,0,1,1,2,4,0}','int4'), 1); 20 | cs_extrema 21 | ------------- 22 | int8:{2,10} 23 | (1 row) 24 | 25 | select cs_extrema(cs_parse('{1,2,3,2,1,0,0,1,1,2,4,0}','int4'), -1); 26 | cs_extrema 27 | ------------ 28 | int8:{6} 29 | (1 row) 30 | 31 | select cs_stretch(cs_parse('{1,2,3,4,5}','int4'), cs_parse('{2,4}','int4'), cs_parse('{1.1,2.2}','float8'), 1.0); 32 | cs_stretch 33 | -------------------------- 34 | float8:{1.1,2.2,2.2,1,1} 35 | (1 row) 36 | 37 | select cs_stretch0(cs_parse('{1,2,3,5}','int8'), cs_parse('{2,3,4}','int8'), cs_parse('{1.1,1.2,1.3}','float4'), 0.0); 38 | cs_stretch0 39 | -------------------------- 40 | float4:{0,1.1,1.2,1.3,0} 41 | (1 row) 42 | 43 | select cs_asof_join(cs_parse('{4,9}','int4'), cs_parse('{1,3,6,10}','int4'), cs_parse('{0.1,0.3,0.6,1.0}','float8')); 44 | cs_asof_join 45 | ---------------- 46 | float8:{0.3,1} 47 | (1 row) 48 | 49 | select cs_asof_join_pos('int8:{4,9}', 'int8:{1,3,6,10}'); 50 | cs_asof_join_pos 51 | ------------------ 52 | int8:{2,2} 53 | (1 row) 54 | 55 | select (Quote_project(q.*,'date:{31-Oct-2013,03-Nov-2013}'->Day)).* from Quote_get('IBM') q; 56 | symbol | day | open | high | low | close | volume 57 | --------+------------+------+------+-----+-------+-------- 58 | IBM | 11-01-2013 | 10.2 | 11 | 10 | 10.5 | 100 59 | IBM | 11-04-2013 | 30.5 | 31 | 30 | 30.2 | 300 60 | (2 rows) 61 | 62 | select (Quote_project(q.*,'date:{1-Nov-2013,2-Nov-2013,03-Nov-2013,04-Nov-2013}'<->Day)).* from Quote_get('IBM') q; 63 | symbol | day | open | high | low | close | volume 64 | --------+------------+------+------+------+-------+-------- 65 | IBM | 11-01-2013 | 10.2 | 11 | 10 | 10.5 | 100 66 | IBM | 11-02-2013 | 20.2 | 20.2 | 20.2 | 20.2 | 200 67 | IBM | 11-04-2013 | 30.5 | 31 | 30 | 30.2 | 300 68 | (3 rows) 69 | 70 | select cs_join('date:{1-Nov-2013,2-Nov-2013,03-Nov-2013,04-Nov-2013}', Day, Close) from Quote_get('IBM'); 71 | cs_join 72 | ------------------------- 73 | float4:{10.5,20.2,30.2} 74 | (1 row) 75 | 76 | select (Quote_project(Quote_get('IBM'), Quote_join('IBM', 'date:{2-Nov-2013,4-Nov-2013,03-Nov-2013,01-Nov-2013}'))).*; 77 | symbol | day | open | high | low | close | volume 78 | --------+------------+------+------+------+-------+-------- 79 | IBM | 11-02-2013 | 20.2 | 20.2 | 20.2 | 20.2 | 200 80 | IBM | 11-04-2013 | 30.5 | 31 | 30 | 30.2 | 300 81 | IBM | 11-04-2013 | 30.5 | 31 | 30 | 30.2 | 300 82 | IBM | 11-01-2013 | 10.2 | 11 | 10 | 10.5 | 100 83 | (4 rows) 84 | 85 | select (Quote_project(Quote_get('IBM'), Quote_join('IBM', 'date:{2-Nov-2013,4-Nov-2013,03-Nov-2013,01-Nov-2013}', 1))).*; 86 | symbol | day | open | high | low | close | volume 87 | --------+------------+------+------+------+-------+-------- 88 | IBM | 11-02-2013 | 20.2 | 20.2 | 20.2 | 20.2 | 200 89 | IBM | 11-04-2013 | 30.5 | 31 | 30 | 30.2 | 300 90 | IBM | 11-04-2013 | 30.5 | 31 | 30 | 30.2 | 300 91 | IBM | 11-01-2013 | 10.2 | 11 | 10 | 10.5 | 100 92 | (4 rows) 93 | 94 | select (Quote_project(Quote_get('IBM'), Quote_join('IBM', 'date:{2-Nov-2013,4-Nov-2013,03-Nov-2013,01-Nov-2013}', 0))).*; 95 | ERROR: no matching timestamp in timeseries 96 | select (Quote_project(Quote_get('IBM'), Quote_join('IBM', 'date:{2-Nov-2013,4-Nov-2013,03-Nov-2013,01-Nov-2013}', -1))).*; 97 | symbol | day | open | high | low | close | volume 98 | --------+------------+------+------+------+-------+-------- 99 | IBM | 11-02-2013 | 20.2 | 20.2 | 20.2 | 20.2 | 200 100 | IBM | 11-04-2013 | 30.5 | 31 | 30 | 30.2 | 300 101 | IBM | 11-02-2013 | 20.2 | 20.2 | 20.2 | 20.2 | 200 102 | IBM | 11-01-2013 | 10.2 | 11 | 10 | 10.5 | 100 103 | (4 rows) 104 | 105 | -------------------------------------------------------------------------------- /expected/transform.out: -------------------------------------------------------------------------------- 1 | select cs_cast(Volume, 'float4') from Quote_get('IBM'); 2 | cs_cast 3 | ------------------------------ 4 | float4:{100,200,300,400,500} 5 | (1 row) 6 | 7 | select cs_iif(Volume>10, Open, Close) from Quote_get('IBM'); 8 | cs_iif 9 | ----------------------------------- 10 | float4:{10.2,20.2,30.5,40.5,50.2} 11 | (1 row) 12 | 13 | select cs_if(Open Close, Day) from Quote_get('IBM'); 44 | cs_filter 45 | ------------------------------ 46 | date:{11-04-2013,11-05-2013} 47 | (1 row) 48 | 49 | select Quote_project(q.*, cs_filter_pos(High > Low*1.01)) from Quote_get('IBM', null, date('05-Nov-2013')) q; 50 | quote_project 51 | -------------------------------------- 52 | (IBM,11-01-2013,10.2,11,10,10.5,100) 53 | (IBM,11-04-2013,30.5,31,30,30.2,300) 54 | (IBM,11-05-2013,40.5,41,40,40.2,400) 55 | (3 rows) 56 | 57 | select Quote_project(q.*, cs_filter_first_pos(High > Low*1.01, 3)) from Quote_get('IBM') q; 58 | quote_project 59 | -------------------------------------- 60 | (IBM,11-01-2013,10.2,11,10,10.5,100) 61 | (IBM,11-04-2013,30.5,31,30,30.2,300) 62 | (IBM,11-05-2013,40.5,41,40,40.2,400) 63 | (3 rows) 64 | 65 | select cs_unique(Volume/200) from Quote_get('IBM'); 66 | cs_unique 67 | -------------- 68 | int4:{0,1,2} 69 | (1 row) 70 | 71 | select cs_reverse(Day) from Quote_get('IBM'); 72 | cs_reverse 73 | --------------------------------------------------------------- 74 | date:{11-06-2013,11-05-2013,11-04-2013,11-02-2013,11-01-2013} 75 | (1 row) 76 | 77 | select cs_diff(Close) from Quote_get('IBM'); 78 | cs_diff 79 | --------------------------- 80 | float4:{0,9.7,10,10,10.3} 81 | (1 row) 82 | 83 | select cs_trend(Close) from Quote_get('IBM'); 84 | cs_trend 85 | ------------------ 86 | char:{0,1,1,1,1} 87 | (1 row) 88 | 89 | select cs_project(q.*) from (select Day,cs_maxof(Open,Close) from Quote_get('IBM', date('02-Nov-2013'))) q; 90 | cs_project 91 | ------------------- 92 | (11-02-2013,20.2) 93 | (11-04-2013,30.5) 94 | (11-05-2013,40.5) 95 | (11-06-2013,50.5) 96 | (4 rows) 97 | 98 | select cs_map(Volume, cs_top_max_pos(Close, 1)) from Quote_get('IBM'); 99 | cs_map 100 | ------------ 101 | int4:{500} 102 | (1 row) 103 | 104 | select cs_union(ibm.Day, abb.Day) from Quote_get('ABB') as abb, Quote_get('IBM') as ibm; 105 | cs_union 106 | -------------------------------------------------------------------------- 107 | date:{11-01-2013,11-02-2013,11-03-2013,11-04-2013,11-05-2013,11-06-2013} 108 | (1 row) 109 | 110 | create function mul2(x float) returns float as $$ begin return x*2; end; $$ language plpgsql strict immutable; 111 | select cs_call(Close, 'mul2'::regproc) from Quote_get('IBM'); 112 | cs_call 113 | -------------------------------------------------------------------- 114 | float8:{21,40.4000015258789,60.4000015258789,80.4000015258789,101} 115 | (1 row) 116 | 117 | select cs_call(Close, 'sin'::regproc) from Quote_get('IBM'); 118 | cs_call 119 | ----------------------------------------------------------------------------------------------------- 120 | float8:{-0.87969575997167,0.975820684524937,-0.937691475202754,0.597759755275722,0.232373761655485} 121 | (1 row) 122 | 123 | select cs_to_int4_array('int4:{1,2,3,4,5}'); 124 | cs_to_int4_array 125 | ------------------ 126 | {1,2,3,4,5} 127 | (1 row) 128 | 129 | select cs_to_float4_array('int4:{1,2,3,4,5}'); 130 | ERROR: Type of sequence element int4 doesn't match with function cs_to_float4_array return type 131 | select cs_sum(cs_from_array(array[1,2,3,4,5])); 132 | cs_sum 133 | -------- 134 | 15 135 | (1 row) 136 | 137 | select cs_to_bpchar_array('bpchar10:{Hello,World,!}'); 138 | cs_to_bpchar_array 139 | -------------------- 140 | {Hello,World,!} 141 | (1 row) 142 | 143 | select cs_from_array(array['Hello','World','!'], 10); 144 | cs_from_array 145 | -------------------------- 146 | bpchar10:{Hello,World,!} 147 | (1 row) 148 | 149 | select cs_parse('{100.99,99.01,"$1,000,000"}', 'money'); 150 | cs_parse 151 | ---------------------------------------- 152 | money:{$100.99,$99.01,"$1,000,000.00"} 153 | (1 row) 154 | 155 | select cs_cast(cs_parse('{100.99,99.01,1000000}', 'money') * 2, 'money'); 156 | cs_cast 157 | ----------------------------------------- 158 | money:{$201.98,$198.02,"$2,000,000.00"} 159 | (1 row) 160 | 161 | select cs_cast('float4:{100.99,99.01,1000000}', 'money'); 162 | cs_cast 163 | ---------------------------------------- 164 | money:{$100.99,$99.01,"$1,000,000.00"} 165 | (1 row) 166 | 167 | select cs_cast('money:{100.99,99.01,"1,000,000"}', 'float8'); 168 | cs_cast 169 | ------------------------------- 170 | float8:{100.99,99.01,1000000} 171 | (1 row) 172 | 173 | select cs_trend('int4:{1,2,3,3,2,2,4,5,6,5,5}'); 174 | cs_trend 175 | ---------------------------------- 176 | char:{0,1,1,1,-1,-1,1,1,1,-1,-1} 177 | (1 row) 178 | 179 | -------------------------------------------------------------------------------- /expected/windowagg.out: -------------------------------------------------------------------------------- 1 | select cs_window_max(Close,3) from Quote_get('IBM'); 2 | cs_window_max 3 | ----------------------------------- 4 | float4:{10.5,20.2,30.2,40.2,50.5} 5 | (1 row) 6 | 7 | select cs_window_min(Close,3) from Quote_get('IBM'); 8 | cs_window_min 9 | ----------------------------- 10 | float4:{0,0,10.5,20.2,30.2} 11 | (1 row) 12 | 13 | select cs_window_sum(Close,3) from Quote_get('IBM'); 14 | cs_window_sum 15 | ----------------------------------------------------------------------------------- 16 | float8:{10.5,30.7000007629395,60.9000015258789,90.6000022888184,120.900001525879} 17 | (1 row) 18 | 19 | select cs_window_avg(Close,3) from Quote_get('IBM'); 20 | cs_window_avg 21 | ---------------------------------------------------------------------------------- 22 | float8:{3.5,10.2333335876465,20.3000005086263,30.2000007629395,40.3000005086263} 23 | (1 row) 24 | 25 | select cs_window_var(Close,3) from Quote_get('IBM'); 26 | cs_window_var 27 | ----------------------------------------------------------------------------------- 28 | float8:{24.5,68.0422272915312,64.6866716512046,66.6666666666667,68.6866614786786} 29 | (1 row) 30 | 31 | select cs_window_dev(Close,3) from Quote_get('IBM'); 32 | cs_window_dev 33 | ----------------------------------------------------------------------------------------------- 34 | float8:{4.94974746830583,8.24877125949866,8.04280247495887,8.16496580927726,8.28774163923313} 35 | (1 row) 36 | 37 | --- Leave only elements corresponding to entire window 38 | select cs_limit(cs_window_max(Close,3), 2) from Quote_get('IBM'); 39 | cs_limit 40 | ------------------------- 41 | float4:{30.2,40.2,50.5} 42 | (1 row) 43 | 44 | select cs_window_min(Close,3) << 2 from Quote_get('IBM'); 45 | ?column? 46 | ------------------------- 47 | float4:{10.5,20.2,30.2} 48 | (1 row) 49 | 50 | select cs_window_sum(Close,3) << 2 from Quote_get('IBM'); 51 | ?column? 52 | ------------------------------------------------------------- 53 | float8:{60.9000015258789,90.6000022888184,120.900001525879} 54 | (1 row) 55 | 56 | select cs_window_avg(Close,3) << 2 from Quote_get('IBM'); 57 | ?column? 58 | ------------------------------------------------------------- 59 | float8:{20.3000005086263,30.2000007629395,40.3000005086263} 60 | (1 row) 61 | 62 | select cs_window_var(Close,3) << 2 from Quote_get('IBM'); 63 | ?column? 64 | ------------------------------------------------------------- 65 | float8:{64.6866716512046,66.6666666666667,68.6866614786786} 66 | (1 row) 67 | 68 | select cs_window_dev(Close,3) << 2 from Quote_get('IBM'); 69 | ?column? 70 | ------------------------------------------------------------- 71 | float8:{8.04280247495887,8.16496580927726,8.28774163923313} 72 | (1 row) 73 | 74 | --- 2-days EMA (Exponential Moving Average) 75 | select cs_window_ema(Close,3) from Quote_get('IBM'); 76 | cs_window_ema 77 | --------------------------------------------------------------------------------- 78 | float8:{10.5,15.3500003814697,22.7750005722046,31.487500667572,40.993750333786} 79 | (1 row) 80 | 81 | --- 2-days ATR (Average True Range) 82 | select cs_window_atr(cs_maxof(High-Low,cs_concat('float4:{0}',cs_maxof(cs_abs((High<<1) - Close), cs_abs((Low<<1) - Close)))), 3) << 2 from Quote_get('IBM'); 83 | ?column? 84 | ------------------------------------------------------------- 85 | float8:{7.16666666666667,8.37777752346463,9.18518476132993} 86 | (1 row) 87 | 88 | -------------------------------------------------------------------------------- /fileio.c: -------------------------------------------------------------------------------- 1 | #include "imcs.h" 2 | #include "fileio.h" 3 | 4 | #ifdef _WIN32 5 | 6 | imcs_file_h imcs_file_open(char const* path) 7 | { 8 | HANDLE h = CreateFile(path, GENERIC_READ | GENERIC_WRITE, 9 | FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, OPEN_ALWAYS, 0, NULL); 10 | 11 | if (h == INVALID_HANDLE_VALUE) { 12 | imcs_ereport(ERRCODE_UNDEFINED_FILE, "Failed to open file '%s': %d", path, GetLastError()); 13 | } 14 | return h; 15 | } 16 | 17 | bool imcs_file_read(imcs_file_h file, void* buf, size_t size, off_t pos) 18 | { 19 | DWORD readBytes = 0; 20 | OVERLAPPED Overlapped; 21 | Overlapped.Offset = (DWORD)LO_32(pos); 22 | Overlapped.OffsetHigh = (DWORD)HI_32(pos); 23 | Overlapped.hEvent = NULL; 24 | if (ReadFile(file, buf, size, &readBytes, &Overlapped)) 25 | { 26 | if (readBytes != size) { 27 | imcs_ereport(ERRCODE_IO_ERROR, "Read %d bytes from file instead of %ld", readBytes, size); 28 | } 29 | } 30 | else 31 | { 32 | int rc = GetLastError(); 33 | if (rc != ERROR_HANDLE_EOF) { 34 | imcs_ereport(ERRCODE_IO_ERROR, "File read failed: %d", GetLastError()); 35 | } 36 | } 37 | return readBytes != 0; 38 | } 39 | 40 | 41 | void imcs_file_write(imcs_file_h file, void const* buf, size_t size, off_t pos) 42 | { 43 | DWORD writtenBytes; 44 | OVERLAPPED Overlapped; 45 | Overlapped.Offset = (DWORD)LO_32(pos); 46 | Overlapped.OffsetHigh = (DWORD)HI_32(pos); 47 | Overlapped.hEvent = NULL; 48 | if (!WriteFile(file, buf, size, &writtenBytes, &Overlapped)) 49 | { 50 | imcs_ereport(ERRCODE_IO_ERROR, "File write failed: %d", GetLastError()); 51 | } 52 | else if (writtenBytes != size) 53 | { 54 | imcs_ereport(ERRCODE_IO_ERROR, "Write %d bytes from file instead of %ld", writtenBytes, size); 55 | } 56 | } 57 | 58 | void imcs_file_close(imcs_file_h file) 59 | { 60 | if (!CloseHandle(file)) { 61 | imcs_ereport(ERRCODE_IO_ERROR, "File close failed: %d", GetLastError()); 62 | } 63 | } 64 | 65 | #else 66 | 67 | #include 68 | #include 69 | #include 70 | #include 71 | #include 72 | 73 | #ifndef O_LARGEFILE 74 | #define O_LARGEFILE 0 75 | #endif 76 | 77 | imcs_file_h imcs_file_open(char const* path) 78 | { 79 | int rc = open(path, O_LARGEFILE | O_RDWR | O_CREAT, 0600); 80 | if (rc < 0) { 81 | imcs_ereport(ERRCODE_UNDEFINED_FILE, "Failed to open file '%s': %d", path, errno); 82 | } 83 | return rc; 84 | } 85 | 86 | bool imcs_file_read(imcs_file_h file, void* buf, size_t size, off_t pos) 87 | { 88 | ssize_t rc = pread(file, buf, size, pos); 89 | if (rc != 0 && rc != size) { 90 | imcs_ereport(ERRCODE_IO_ERROR, "File read failed: %d", errno); 91 | } 92 | return rc != 0; 93 | } 94 | 95 | 96 | void imcs_file_write(imcs_file_h file, void const* buf, size_t size, off_t pos) 97 | { 98 | ssize_t rc = pwrite(file, buf, size, pos); 99 | if (rc != size) { 100 | imcs_ereport(ERRCODE_IO_ERROR, "File write failed: %d", errno); 101 | } 102 | } 103 | 104 | void imcs_file_close(imcs_file_h file) 105 | { 106 | if (close(file) < 0) { 107 | imcs_ereport(ERRCODE_IO_ERROR, "File close failed: %d", errno); 108 | } 109 | } 110 | 111 | #endif 112 | -------------------------------------------------------------------------------- /fileio.h: -------------------------------------------------------------------------------- 1 | #ifndef __FILEIO_H__ 2 | #define __FILEIO_H__ 3 | 4 | #include 5 | #include 6 | 7 | /** 8 | * File IO: access to file system 9 | * I can not use PostgreSQL fd.h, because I need reentrant read/write 10 | */ 11 | 12 | #ifdef _WIN32 13 | typedef void* imcs_file_h; 14 | #else 15 | typedef int imcs_file_h; 16 | #endif 17 | 18 | imcs_file_h imcs_file_open(char const* path); 19 | bool imcs_file_read(imcs_file_h file, void* buf, size_t size, off_t pos); 20 | void imcs_file_write(imcs_file_h file, void const* buf, size_t size, off_t pos); 21 | void imcs_file_close(imcs_file_h file); 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /func.h: -------------------------------------------------------------------------------- 1 | #ifndef __FUNC_H__ 2 | #define __FUNC_H__ 3 | 4 | #include "imcs.h" 5 | 6 | /* Functions defined for all scalar types */ 7 | #define IMCS_FUNC_DECL(TYPE) \ 8 | typedef TYPE(*imcs_func_##TYPE##_ptr_t)(TYPE arg); \ 9 | typedef TYPE(*imcs_func2_##TYPE##_ptr_t)(TYPE arg1, TYPE arg2); \ 10 | bool imcs_next_##TYPE(imcs_iterator_h iterator, TYPE* val); \ 11 | imcs_iterator_h imcs_parse_##TYPE(char const* str, int elem_size); \ 12 | imcs_iterator_h imcs_adt_parse_##TYPE(char const* str, imcs_adt_parser_t* parser); \ 13 | imcs_iterator_h imcs_const_##TYPE(TYPE val); \ 14 | imcs_iterator_h imcs_func_##TYPE(imcs_iterator_h input, imcs_func_##TYPE##_ptr_t func); \ 15 | imcs_iterator_h imcs_func2_##TYPE(imcs_iterator_h left, imcs_iterator_h right, imcs_func2_##TYPE##_ptr_t func); \ 16 | imcs_iterator_h imcs_map_##TYPE(imcs_iterator_h input, imcs_iterator_h positions); \ 17 | imcs_iterator_h imcs_union_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 18 | imcs_iterator_h imcs_add_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 19 | imcs_iterator_h imcs_sub_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 20 | imcs_iterator_h imcs_mul_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 21 | imcs_iterator_h imcs_div_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 22 | imcs_iterator_h imcs_pow_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 23 | imcs_iterator_h imcs_mod_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 24 | imcs_iterator_h imcs_maxof_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 25 | imcs_iterator_h imcs_minof_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 26 | imcs_iterator_h imcs_eq_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 27 | imcs_iterator_h imcs_ne_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 28 | imcs_iterator_h imcs_gt_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 29 | imcs_iterator_h imcs_ge_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 30 | imcs_iterator_h imcs_lt_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 31 | imcs_iterator_h imcs_le_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 32 | imcs_iterator_h imcs_wsum_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 33 | imcs_iterator_h imcs_wavg_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 34 | imcs_iterator_h imcs_cov_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 35 | imcs_iterator_h imcs_corr_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 36 | imcs_iterator_h imcs_norm_##TYPE(imcs_iterator_h input); \ 37 | imcs_iterator_h imcs_thin_##TYPE(imcs_iterator_h input, size_t origin, size_t step); \ 38 | imcs_iterator_h imcs_iif_##TYPE(imcs_iterator_h cond, imcs_iterator_h then_iter, imcs_iterator_h else_iter); \ 39 | imcs_iterator_h imcs_if_##TYPE(imcs_iterator_h cond, imcs_iterator_h then_iter, imcs_iterator_h else_iter); \ 40 | imcs_iterator_h imcs_filter_##TYPE(imcs_iterator_h cond, imcs_iterator_h input); \ 41 | imcs_iterator_h imcs_unique_##TYPE(imcs_iterator_h input); \ 42 | imcs_iterator_h imcs_isnan_##TYPE(imcs_iterator_h input); \ 43 | imcs_iterator_h imcs_abs_##TYPE(imcs_iterator_h input); \ 44 | imcs_iterator_h imcs_abs_##TYPE(imcs_iterator_h input); \ 45 | imcs_iterator_h imcs_neg_##TYPE(imcs_iterator_h input); \ 46 | imcs_iterator_h imcs_reverse_##TYPE(imcs_iterator_h input); \ 47 | imcs_iterator_h imcs_diff_##TYPE(imcs_iterator_h input); \ 48 | imcs_iterator_h imcs_trend_##TYPE(imcs_iterator_h input); \ 49 | imcs_iterator_h imcs_max_##TYPE(imcs_iterator_h input); \ 50 | imcs_iterator_h imcs_min_##TYPE(imcs_iterator_h input); \ 51 | imcs_iterator_h imcs_sum_##TYPE(imcs_iterator_h input); \ 52 | imcs_iterator_h imcs_any_##TYPE(imcs_iterator_h input); \ 53 | imcs_iterator_h imcs_all_##TYPE(imcs_iterator_h input); \ 54 | imcs_iterator_h imcs_prd_##TYPE(imcs_iterator_h input); \ 55 | imcs_iterator_h imcs_avg_##TYPE(imcs_iterator_h input); \ 56 | imcs_iterator_h imcs_var_##TYPE(imcs_iterator_h input); \ 57 | imcs_iterator_h imcs_dev_##TYPE(imcs_iterator_h input); \ 58 | imcs_iterator_h imcs_median_##TYPE(imcs_iterator_h input); \ 59 | imcs_iterator_h imcs_group_max_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 60 | imcs_iterator_h imcs_group_min_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 61 | imcs_iterator_h imcs_group_sum_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 62 | imcs_iterator_h imcs_group_any_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 63 | imcs_iterator_h imcs_group_all_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 64 | imcs_iterator_h imcs_group_avg_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 65 | imcs_iterator_h imcs_group_var_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 66 | imcs_iterator_h imcs_group_dev_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 67 | imcs_iterator_h imcs_group_last_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 68 | imcs_iterator_h imcs_group_first_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 69 | imcs_iterator_h imcs_win_group_max_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 70 | imcs_iterator_h imcs_win_group_min_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 71 | imcs_iterator_h imcs_win_group_sum_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 72 | imcs_iterator_h imcs_win_group_any_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 73 | imcs_iterator_h imcs_win_group_all_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 74 | imcs_iterator_h imcs_win_group_avg_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 75 | imcs_iterator_h imcs_win_group_var_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 76 | imcs_iterator_h imcs_win_group_dev_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 77 | imcs_iterator_h imcs_win_group_last_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 78 | imcs_iterator_h imcs_win_group_first_##TYPE(imcs_iterator_h input, imcs_iterator_h group_by); \ 79 | imcs_iterator_h imcs_grid_max_##TYPE(imcs_iterator_h input, size_t interval); \ 80 | imcs_iterator_h imcs_grid_min_##TYPE(imcs_iterator_h input, size_t interval); \ 81 | imcs_iterator_h imcs_grid_sum_##TYPE(imcs_iterator_h input, size_t interval); \ 82 | imcs_iterator_h imcs_grid_avg_##TYPE(imcs_iterator_h input, size_t interval); \ 83 | imcs_iterator_h imcs_grid_var_##TYPE(imcs_iterator_h input, size_t interval); \ 84 | imcs_iterator_h imcs_grid_dev_##TYPE(imcs_iterator_h input, size_t interval); \ 85 | imcs_iterator_h imcs_window_max_##TYPE(imcs_iterator_h input, size_t interval); \ 86 | imcs_iterator_h imcs_window_min_##TYPE(imcs_iterator_h input, size_t interval); \ 87 | imcs_iterator_h imcs_window_sum_##TYPE(imcs_iterator_h input, size_t interval); \ 88 | imcs_iterator_h imcs_window_avg_##TYPE(imcs_iterator_h input, size_t interval); \ 89 | imcs_iterator_h imcs_window_var_##TYPE(imcs_iterator_h input, size_t interval); \ 90 | imcs_iterator_h imcs_window_dev_##TYPE(imcs_iterator_h input, size_t interval); \ 91 | imcs_iterator_h imcs_window_ema_##TYPE(imcs_iterator_h input, size_t interval); \ 92 | imcs_iterator_h imcs_window_atr_##TYPE(imcs_iterator_h input, size_t interval); \ 93 | void imcs_hash_max_##TYPE(imcs_iterator_h result[2], imcs_iterator_h input, imcs_iterator_h group_by); \ 94 | void imcs_hash_min_##TYPE(imcs_iterator_h result[2], imcs_iterator_h input, imcs_iterator_h group_by); \ 95 | void imcs_hash_sum_##TYPE(imcs_iterator_h result[2], imcs_iterator_h input, imcs_iterator_h group_by); \ 96 | void imcs_hash_any_##TYPE(imcs_iterator_h result[2], imcs_iterator_h input, imcs_iterator_h group_by); \ 97 | void imcs_hash_all_##TYPE(imcs_iterator_h result[2], imcs_iterator_h input, imcs_iterator_h group_by); \ 98 | void imcs_hash_avg_##TYPE(imcs_iterator_h result[2], imcs_iterator_h input, imcs_iterator_h group_by); \ 99 | imcs_iterator_h imcs_top_max_##TYPE(imcs_iterator_h input, size_t top); \ 100 | imcs_iterator_h imcs_top_min_##TYPE(imcs_iterator_h input, size_t top); \ 101 | imcs_iterator_h imcs_top_max_pos_##TYPE(imcs_iterator_h input, size_t top); \ 102 | imcs_iterator_h imcs_top_min_pos_##TYPE(imcs_iterator_h input, size_t top); \ 103 | imcs_iterator_h imcs_cum_avg_##TYPE(imcs_iterator_h input); \ 104 | imcs_iterator_h imcs_cum_sum_##TYPE(imcs_iterator_h input); \ 105 | imcs_iterator_h imcs_cum_min_##TYPE(imcs_iterator_h input); \ 106 | imcs_iterator_h imcs_cum_max_##TYPE(imcs_iterator_h input); \ 107 | imcs_iterator_h imcs_cum_prd_##TYPE(imcs_iterator_h input); \ 108 | imcs_iterator_h imcs_cum_var_##TYPE(imcs_iterator_h input); \ 109 | imcs_iterator_h imcs_cum_dev_##TYPE(imcs_iterator_h input); \ 110 | imcs_iterator_h imcs_histogram_##TYPE(imcs_iterator_h input, TYPE min_value, TYPE max_value, size_t n_intervals); \ 111 | imcs_iterator_h imcs_cross_##TYPE(imcs_iterator_h input, int first_cross_direction); \ 112 | imcs_iterator_h imcs_extrema_##TYPE(imcs_iterator_h input, int first_extremum); \ 113 | imcs_iterator_h imcs_repeat_##TYPE(imcs_iterator_h input, int n_times); \ 114 | imcs_iterator_h imcs_not_##TYPE(imcs_iterator_h input); \ 115 | imcs_iterator_h imcs_bit_not_##TYPE(imcs_iterator_h input); \ 116 | imcs_iterator_h imcs_and_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 117 | imcs_iterator_h imcs_or_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 118 | imcs_iterator_h imcs_xor_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 119 | imcs_iterator_h imcs_asof_join_pos_##TYPE(imcs_iterator_h left, imcs_iterator_h right); \ 120 | imcs_iterator_h imcs_join_pos_##TYPE(imcs_iterator_h left, imcs_iterator_h right);\ 121 | imcs_iterator_h imcs_join_unsorted_##TYPE(imcs_timeseries_t* ts, imcs_iterator_h join_with, int direction); \ 122 | imcs_iterator_h imcs_rank_##TYPE(imcs_iterator_h, imcs_order_t order); \ 123 | imcs_iterator_h imcs_dense_rank_##TYPE(imcs_iterator_h, imcs_order_t order); \ 124 | imcs_iterator_h imcs_sort_##TYPE(imcs_iterator_h, imcs_order_t order); \ 125 | imcs_iterator_h imcs_sort_pos_##TYPE(imcs_iterator_h, imcs_order_t order); \ 126 | imcs_iterator_h imcs_quantile_##TYPE(imcs_iterator_h, size_t q_num) \ 127 | 128 | 129 | IMCS_FUNC_DECL(int8); 130 | IMCS_FUNC_DECL(int16); 131 | IMCS_FUNC_DECL(int32); 132 | IMCS_FUNC_DECL(int64); 133 | IMCS_FUNC_DECL(float); 134 | IMCS_FUNC_DECL(double); 135 | 136 | 137 | #define IMCS_FUNC2_DECL(MNEM, TYPE1, TYPE2) \ 138 | imcs_iterator_h imcs_##MNEM##_##TYPE1##_##TYPE2(imcs_iterator_h ts1, imcs_iterator_h ts2, imcs_iterator_h values, TYPE2 filler) 139 | 140 | IMCS_FUNC2_DECL(stretch, int32, int8); 141 | IMCS_FUNC2_DECL(stretch, int32, int16); 142 | IMCS_FUNC2_DECL(stretch, int32, int32); 143 | IMCS_FUNC2_DECL(stretch, int32, int64); 144 | IMCS_FUNC2_DECL(stretch, int32, float); 145 | IMCS_FUNC2_DECL(stretch, int32, double); 146 | IMCS_FUNC2_DECL(stretch, int64, int8); 147 | IMCS_FUNC2_DECL(stretch, int64, int16); 148 | IMCS_FUNC2_DECL(stretch, int64, int32); 149 | IMCS_FUNC2_DECL(stretch, int64, int64); 150 | IMCS_FUNC2_DECL(stretch, int64, float); 151 | IMCS_FUNC2_DECL(stretch, int64, double); 152 | 153 | IMCS_FUNC2_DECL(stretch0, int32, int8); 154 | IMCS_FUNC2_DECL(stretch0, int32, int16); 155 | IMCS_FUNC2_DECL(stretch0, int32, int32); 156 | IMCS_FUNC2_DECL(stretch0, int32, int64); 157 | IMCS_FUNC2_DECL(stretch0, int32, float); 158 | IMCS_FUNC2_DECL(stretch0, int32, double); 159 | IMCS_FUNC2_DECL(stretch0, int64, int8); 160 | IMCS_FUNC2_DECL(stretch0, int64, int16); 161 | IMCS_FUNC2_DECL(stretch0, int64, int32); 162 | IMCS_FUNC2_DECL(stretch0, int64, int64); 163 | IMCS_FUNC2_DECL(stretch0, int64, float); 164 | IMCS_FUNC2_DECL(stretch0, int64, double); 165 | 166 | #define IMCS_JOIN_DECL(JOIN,TYPE1, TYPE2) \ 167 | imcs_iterator_h imcs_##JOIN##_##TYPE1##_##TYPE2(imcs_iterator_h ts1, imcs_iterator_h ts2, imcs_iterator_h values) 168 | 169 | IMCS_JOIN_DECL(join, int32, int8); 170 | IMCS_JOIN_DECL(join, int32, int16); 171 | IMCS_JOIN_DECL(join, int32, int32); 172 | IMCS_JOIN_DECL(join, int32, int64); 173 | IMCS_JOIN_DECL(join, int32, float); 174 | IMCS_JOIN_DECL(join, int32, double); 175 | IMCS_JOIN_DECL(join, int64, int8); 176 | IMCS_JOIN_DECL(join, int64, int16); 177 | IMCS_JOIN_DECL(join, int64, int32); 178 | IMCS_JOIN_DECL(join, int64, int64); 179 | IMCS_JOIN_DECL(join, int64, float); 180 | IMCS_JOIN_DECL(join, int64, double); 181 | 182 | IMCS_JOIN_DECL(asof_join, int32, int8); 183 | IMCS_JOIN_DECL(asof_join, int32, int16); 184 | IMCS_JOIN_DECL(asof_join, int32, int32); 185 | IMCS_JOIN_DECL(asof_join, int32, int64); 186 | IMCS_JOIN_DECL(asof_join, int32, float); 187 | IMCS_JOIN_DECL(asof_join, int32, double); 188 | IMCS_JOIN_DECL(asof_join, int64, int8); 189 | IMCS_JOIN_DECL(asof_join, int64, int16); 190 | IMCS_JOIN_DECL(asof_join, int64, int32); 191 | IMCS_JOIN_DECL(asof_join, int64, int64); 192 | IMCS_JOIN_DECL(asof_join, int64, float); 193 | IMCS_JOIN_DECL(asof_join, int64, double); 194 | 195 | /* Functions for sequence of char */ 196 | bool imcs_next_char(imcs_iterator_h iterator, void* val); 197 | imcs_iterator_h imcs_parse_char(char const* val, int elem_size); 198 | imcs_iterator_h imcs_const_char(void const* val, size_t size); 199 | imcs_iterator_h imcs_iif_char(imcs_iterator_h cond, imcs_iterator_h then_iter, imcs_iterator_h else_iter); 200 | imcs_iterator_h imcs_if_char(imcs_iterator_h cond, imcs_iterator_h then_iter, imcs_iterator_h else_iter); 201 | imcs_iterator_h imcs_filter_char(imcs_iterator_h cond, imcs_iterator_h input); 202 | imcs_iterator_h imcs_unique_char(imcs_iterator_h input); 203 | imcs_iterator_h imcs_thin_char(imcs_iterator_h input, size_t origin, size_t step); 204 | imcs_iterator_h imcs_repeat_char(imcs_iterator_h input, int count); 205 | imcs_iterator_h imcs_map_char(imcs_iterator_h left, imcs_iterator_h right); 206 | imcs_iterator_h imcs_reverse_char(imcs_iterator_h input); 207 | imcs_iterator_h imcs_add_char(imcs_iterator_h left, imcs_iterator_h right); 208 | imcs_iterator_h imcs_eq_char(imcs_iterator_h left, imcs_iterator_h right); 209 | imcs_iterator_h imcs_ne_char(imcs_iterator_h left, imcs_iterator_h right); 210 | imcs_iterator_h imcs_gt_char(imcs_iterator_h left, imcs_iterator_h right); 211 | imcs_iterator_h imcs_ge_char(imcs_iterator_h left, imcs_iterator_h right); 212 | imcs_iterator_h imcs_lt_char(imcs_iterator_h left, imcs_iterator_h right); 213 | imcs_iterator_h imcs_le_char(imcs_iterator_h left, imcs_iterator_h right); 214 | 215 | /* Polymorphic functions */ 216 | imcs_iterator_h imcs_concat(imcs_iterator_h left, imcs_iterator_h right); 217 | imcs_iterator_h imcs_cat(imcs_iterator_h left, imcs_iterator_h right); 218 | imcs_iterator_h imcs_limit(imcs_iterator_h input, imcs_pos_t from, imcs_pos_t till); 219 | imcs_iterator_h imcs_count_iterator(imcs_iterator_h input); 220 | imcs_iterator_h imcs_approxdc(imcs_iterator_h input); 221 | imcs_iterator_h imcs_group_count(imcs_iterator_h group_by); 222 | imcs_iterator_h imcs_group_approxdc(imcs_iterator_h input, imcs_iterator_h group_by); 223 | void imcs_hash_count(imcs_iterator_h result[2], imcs_iterator_h group_by); 224 | void imcs_hash_approxdc(imcs_iterator_h result[2], imcs_iterator_h input, imcs_iterator_h group_by); 225 | void imcs_hash_dup_count(imcs_iterator_h result[2], imcs_iterator_h input, imcs_iterator_h group_by, size_t min_occurrences); 226 | 227 | imcs_iterator_h imcs_like(imcs_iterator_h input, char const* pattern); 228 | imcs_iterator_h imcs_ilike(imcs_iterator_h input, char const* pattern); 229 | 230 | imcs_iterator_h imcs_filter_pos(imcs_iterator_h cond); 231 | imcs_iterator_h imcs_filter_first_pos(imcs_iterator_h cond, size_t n); 232 | void imcs_tee(imcs_iterator_h out_iterators[2], imcs_iterator_h in_iterator); 233 | 234 | void imcs_from_array(imcs_iterator_h result, void const* buf, size_t buf_size); 235 | void imcs_to_array(imcs_iterator_h iterator, void* buf, size_t buf_size); 236 | 237 | imcs_iterator_h imcs_cast_to_char(imcs_iterator_h iterator, int elem_size); 238 | 239 | #define IMCS_CAST_DECL(FROM_TYPE, TO_TYPE) \ 240 | imcs_iterator_h imcs_##TO_TYPE##_from_##FROM_TYPE(imcs_iterator_h input) 241 | 242 | #define IMCS_CASTS_DECL(TYPE) \ 243 | IMCS_CAST_DECL(TYPE, int8); \ 244 | IMCS_CAST_DECL(TYPE, int16); \ 245 | IMCS_CAST_DECL(TYPE, int32); \ 246 | IMCS_CAST_DECL(TYPE, int64); \ 247 | IMCS_CAST_DECL(TYPE, float); \ 248 | IMCS_CAST_DECL(TYPE, double) 249 | 250 | IMCS_CASTS_DECL(int8); 251 | IMCS_CASTS_DECL(int16); 252 | IMCS_CASTS_DECL(int32); 253 | IMCS_CASTS_DECL(int64); 254 | IMCS_CASTS_DECL(float); 255 | IMCS_CASTS_DECL(double); 256 | 257 | #define IMCS_CALL_DECL(RET_TYPE, ARG_TYPE) \ 258 | imcs_iterator_h imcs_##RET_TYPE##_call_##ARG_TYPE(imcs_iterator_h input, Oid funcid) 259 | 260 | #define IMCS_CALLS_DECL(TYPE) \ 261 | IMCS_CALL_DECL(TYPE, int8); \ 262 | IMCS_CALL_DECL(TYPE, int16); \ 263 | IMCS_CALL_DECL(TYPE, int32); \ 264 | IMCS_CALL_DECL(TYPE, int64); \ 265 | IMCS_CALL_DECL(TYPE, float); \ 266 | IMCS_CALL_DECL(TYPE, double) 267 | 268 | IMCS_CALLS_DECL(int8); 269 | IMCS_CALLS_DECL(int16); 270 | IMCS_CALLS_DECL(int32); 271 | IMCS_CALLS_DECL(int64); 272 | IMCS_CALLS_DECL(float); 273 | IMCS_CALLS_DECL(double); 274 | 275 | #endif 276 | -------------------------------------------------------------------------------- /imcs.conf: -------------------------------------------------------------------------------- 1 | shared_preload_libraries = 'imcs' 2 | extra_float_digits=0 -------------------------------------------------------------------------------- /imcs.control: -------------------------------------------------------------------------------- 1 | # imcs extension 2 | comment = 'In-Memory Columnar Store' 3 | default_version = '1.1' 4 | module_pathname = '$libdir/imcs' 5 | relocatable = true 6 | -------------------------------------------------------------------------------- /imcs.h: -------------------------------------------------------------------------------- 1 | #ifndef __IMCS_H__ 2 | #define __IMCS_H__ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | struct imcs_page_t_; 21 | typedef struct imcs_page_t_ imcs_page_t; 22 | typedef uint64 imcs_pos_t; 23 | typedef uint64 imcs_count_t; 24 | 25 | extern int imcs_page_size; 26 | extern int imcs_tile_size; 27 | extern int imcs_dict_size; 28 | extern bool imcs_sync_load; 29 | extern bool imcs_use_rle; 30 | extern int imcs_cache_size; 31 | extern char* imcs_file_path; 32 | 33 | #define IMCS_INFINITY (-1) 34 | #define IMCS_MAX_ERROR_MSG_LEN 256 35 | #define IMCS_SMALL_DICTIONARY 0x10000 /* 64kb - identifier can be stored in two bytes */ 36 | 37 | #ifdef __GNUC__ 38 | #define IMCS_ALIGN_16 __attribute__((aligned(16))) 39 | #else 40 | #define IMCS_ALIGN_16 41 | #endif 42 | 43 | typedef union 44 | { 45 | int8 val_int8; 46 | int16 val_int16; 47 | int32 val_int32; 48 | int64 val_int64; 49 | float val_float; 50 | double val_double; 51 | char* val_ptr; 52 | } imcs_key_t; 53 | 54 | typedef union 55 | { 56 | char arr_char[1]; 57 | int8 arr_int8[1]; 58 | int16 arr_int16[1]; 59 | int32 arr_int32[1]; 60 | int64 arr_int64[1]; 61 | float arr_float[1]; 62 | double arr_double[1]; 63 | } imcs_tile_t IMCS_ALIGN_16; 64 | 65 | typedef enum 66 | { 67 | TID_int8, 68 | TID_int16, 69 | TID_int32, 70 | TID_date, 71 | TID_int64, 72 | TID_time, 73 | TID_timestamp, 74 | TID_money, 75 | TID_float, 76 | TID_double, 77 | TID_char 78 | } imcs_elem_typeid_t; 79 | 80 | extern const imcs_elem_typeid_t imcs_underlying_type[]; 81 | extern const char* const imcs_type_mnems[]; 82 | extern const Oid imcs_elem_type_to_oid[]; 83 | 84 | typedef enum { 85 | IMCS_ASC_ORDER, 86 | IMCS_DESC_ORDER 87 | } imcs_order_t; 88 | 89 | struct imcs_iterator_t_; 90 | typedef bool(*imcs_iterator_next_t)(struct imcs_iterator_t_* iterator); 91 | typedef void(*imcs_iterator_reset_t)(struct imcs_iterator_t_* iterator); 92 | typedef bool(*imcs_iterator_prepare_t)(struct imcs_iterator_t_* iterator); 93 | typedef void(*imcs_iterator_merge_t)(struct imcs_iterator_t_* dst, struct imcs_iterator_t_* src); 94 | 95 | /** 96 | * Timeseries header 97 | */ 98 | typedef struct imcs_timeseries_t 99 | { 100 | imcs_page_t* root_page; /* root of B-Tree */ 101 | imcs_elem_typeid_t elem_type; 102 | bool is_timestamp; 103 | int elem_size; 104 | imcs_count_t count; 105 | } imcs_timeseries_t; 106 | 107 | typedef enum 108 | { 109 | FLAG_RANDOM_ACCESS = 1, /* it is timeseries stored as B-Tree, supporting random acess by position */ 110 | FLAG_CONTEXT_FREE = 2, /* each element can be calculated independetly: such timeseries allows concurrent execution */ 111 | FLAG_PREPARED = 4, /* result was already prepared by prepare() function during parallel query execution */ 112 | FLAG_CONSTANT = 8, /* timeries of repeated costant element */ 113 | FLAG_TRANSLATED = 16 /* character element value was replaced with integer identifier using dictionary */ 114 | } imcs_flags_t; 115 | 116 | typedef struct 117 | { 118 | jmp_buf unwind_buf; 119 | int err_code; 120 | char err_msg[IMCS_MAX_ERROR_MSG_LEN]; 121 | } imcs_error_handler_t; 122 | 123 | extern void imcs_ereport(int err_code, char const* err_msg,...) __attribute__((noreturn)); 124 | 125 | typedef struct { 126 | char* val; 127 | size_t len; 128 | } imcs_dict_key_t; 129 | 130 | typedef struct { 131 | imcs_dict_key_t key; 132 | size_t code; 133 | } imcs_dict_entry_t; 134 | 135 | extern imcs_dict_entry_t** imcs_dict_code_map; 136 | 137 | enum imcs_commands 138 | { 139 | imcs_cmd_parse, 140 | imcs_cmd_const, 141 | imcs_cmd_cast, 142 | imcs_cmd_add, 143 | imcs_cmd_mul, 144 | imcs_cmd_sub, 145 | imcs_cmd_div, 146 | imcs_cmd_mod, 147 | imcs_cmd_pow, 148 | imcs_cmd_and, 149 | imcs_cmd_or, 150 | imcs_cmd_xor, 151 | imcs_cmd_concat, 152 | imcs_cmd_cat, 153 | imcs_cmd_eq, 154 | imcs_cmd_ne, 155 | imcs_cmd_ge, 156 | imcs_cmd_le, 157 | imcs_cmd_lt, 158 | imcs_cmd_gt, 159 | imcs_cmd_maxof, 160 | imcs_cmd_minof, 161 | imcs_cmd_neg, 162 | imcs_cmd_not, 163 | imcs_cmd_bit_not, 164 | imcs_cmd_abs, 165 | imcs_cmd_limit, 166 | imcs_cmd_sin, 167 | imcs_cmd_cos, 168 | imcs_cmd_tan, 169 | imcs_cmd_exp, 170 | imcs_cmd_asin, 171 | imcs_cmd_acos, 172 | imcs_cmd_atan, 173 | imcs_cmd_sqrt, 174 | imcs_cmd_log, 175 | imcs_cmd_ceil, 176 | imcs_cmd_floor, 177 | imcs_cmd_isnan, 178 | imcs_cmd_wsum, 179 | imcs_cmd_wavg, 180 | imcs_cmd_corr, 181 | imcs_cmd_cov, 182 | imcs_cmd_norm, 183 | imcs_cmd_thin, 184 | imcs_cmd_iif, 185 | imcs_cmd_if, 186 | imcs_cmd_filter, 187 | imcs_cmd_filter_pos, 188 | imcs_cmd_filter_first_pos, 189 | imcs_cmd_unique, 190 | imcs_cmd_reverse, 191 | imcs_cmd_diff, 192 | imcs_cmd_trend, 193 | imcs_cmd_repeat, 194 | imcs_cmd_count, 195 | imcs_cmd_approxdc, 196 | imcs_cmd_max, 197 | imcs_cmd_min, 198 | imcs_cmd_avg, 199 | imcs_cmd_sum, 200 | imcs_cmd_prd, 201 | imcs_cmd_var, 202 | imcs_cmd_dev, 203 | imcs_cmd_all, 204 | imcs_cmd_any, 205 | imcs_cmd_median, 206 | imcs_cmd_group_count, 207 | imcs_cmd_group_approxdc, 208 | imcs_cmd_group_max, 209 | imcs_cmd_group_min, 210 | imcs_cmd_group_avg, 211 | imcs_cmd_group_sum, 212 | imcs_cmd_group_var, 213 | imcs_cmd_group_dev, 214 | imcs_cmd_group_all, 215 | imcs_cmd_group_any, 216 | imcs_cmd_group_last, 217 | imcs_cmd_group_first, 218 | imcs_cmd_win_group_max, 219 | imcs_cmd_win_group_min, 220 | imcs_cmd_win_group_avg, 221 | imcs_cmd_win_group_sum, 222 | imcs_cmd_win_group_var, 223 | imcs_cmd_win_group_dev, 224 | imcs_cmd_win_group_all, 225 | imcs_cmd_win_group_any, 226 | imcs_cmd_win_group_last, 227 | imcs_cmd_win_group_first, 228 | imcs_cmd_grid_max, 229 | imcs_cmd_grid_min, 230 | imcs_cmd_grid_avg, 231 | imcs_cmd_grid_sum, 232 | imcs_cmd_grid_var, 233 | imcs_cmd_grid_dev, 234 | imcs_cmd_window_max, 235 | imcs_cmd_window_min, 236 | imcs_cmd_window_avg, 237 | imcs_cmd_window_sum, 238 | imcs_cmd_window_var, 239 | imcs_cmd_window_dev, 240 | imcs_cmd_window_ema, 241 | imcs_cmd_window_atr, 242 | imcs_cmd_hash_count, 243 | imcs_cmd_hash_dup_count, 244 | imcs_cmd_hash_max, 245 | imcs_cmd_hash_min, 246 | imcs_cmd_hash_avg, 247 | imcs_cmd_hash_sum, 248 | imcs_cmd_hash_all, 249 | imcs_cmd_hash_any, 250 | imcs_cmd_top_max, 251 | imcs_cmd_top_min, 252 | imcs_cmd_top_max_pos, 253 | imcs_cmd_top_min_pos, 254 | imcs_cmd_cum_max, 255 | imcs_cmd_cum_min, 256 | imcs_cmd_cum_avg, 257 | imcs_cmd_cum_sum, 258 | imcs_cmd_cum_prd, 259 | imcs_cmd_cum_var, 260 | imcs_cmd_cum_dev, 261 | imcs_cmd_histogram, 262 | imcs_cmd_cross, 263 | imcs_cmd_extrema, 264 | imcs_cmd_stretch, 265 | imcs_cmd_stretch0, 266 | imcs_cmd_asof_join, 267 | imcs_cmd_asof_join_pos, 268 | imcs_cmd_join, 269 | imcs_cmd_join_pos, 270 | imcs_cmd_map, 271 | imcs_cmd_union, 272 | imcs_cmd_empty, 273 | imcs_cmd_project, 274 | imcs_cmd_project_agg, 275 | imcs_cmd_year, 276 | imcs_cmd_month, 277 | imcs_cmd_mday, 278 | imcs_cmd_wday, 279 | imcs_cmd_hour, 280 | imcs_cmd_minute, 281 | imcs_cmd_second, 282 | imcs_cmd_week, 283 | imcs_cmd_quarter, 284 | imcs_cmd_call, 285 | imcs_cmd_to_array, 286 | imcs_cmd_from_array, 287 | imcs_cmd_like, 288 | imcs_cmd_ilike, 289 | imcs_cmd_sort, 290 | imcs_cmd_sort_pos, 291 | imcs_cmd_rank, 292 | imcs_cmd_dense_rank, 293 | imcs_cmd_quantile, 294 | imcs_cmd_last_command, 295 | 296 | imcs_cmd_int8_from = imcs_cmd_cast, 297 | imcs_cmd_int16_from = imcs_cmd_cast, 298 | imcs_cmd_int32_from = imcs_cmd_cast, 299 | imcs_cmd_int64_from = imcs_cmd_cast, 300 | imcs_cmd_float_from = imcs_cmd_cast, 301 | imcs_cmd_double_from = imcs_cmd_cast, 302 | 303 | imcs_cmd_stretch_int32 = imcs_cmd_stretch, 304 | imcs_cmd_stretch_int64 = imcs_cmd_stretch, 305 | imcs_cmd_stretch0_int32 = imcs_cmd_stretch0, 306 | imcs_cmd_stretch0_int64 = imcs_cmd_stretch0, 307 | imcs_cmd_asof_join_int32 = imcs_cmd_asof_join, 308 | imcs_cmd_asof_join_int64 = imcs_cmd_asof_join, 309 | imcs_cmd_join_int32 = imcs_cmd_join, 310 | imcs_cmd_join_int64 = imcs_cmd_join, 311 | 312 | imcs_cmd_int8_call = imcs_cmd_call, 313 | imcs_cmd_int16_call = imcs_cmd_call, 314 | imcs_cmd_int32_call = imcs_cmd_call, 315 | imcs_cmd_int64_call = imcs_cmd_call, 316 | imcs_cmd_float_call = imcs_cmd_call, 317 | imcs_cmd_double_call = imcs_cmd_call 318 | }; 319 | 320 | 321 | /* 322 | * Timeseries iterator: it is used not only for iteration but for constructing pipe of different operations with sequence 323 | */ 324 | typedef struct imcs_iterator_t_ 325 | { 326 | imcs_iterator_next_t next; /* method for obtaning next portion of values */ 327 | imcs_iterator_reset_t reset; /* start iteration from beginning */ 328 | imcs_iterator_prepare_t prepare; /* prepare iterator (used to start parallel processing) */ 329 | imcs_iterator_merge_t merge; /* merge two iterators (used to merge results of parallel processing) */ 330 | uint16 elem_size; /* size of element */ 331 | uint16 tile_size; /* number of tile items */ 332 | uint16 tile_offs; /* offset to first not handled tile item */ 333 | uint8 rle_offs; /* index within same RLE value */ 334 | uint8 flags; /* bitmap of imcs_flags_t flags */ 335 | imcs_pos_t first_pos; /* first sequence number (inclusive) */ 336 | imcs_pos_t next_pos; /* sequence number of element returned by sudsequent invocation of next() function */ 337 | imcs_pos_t last_pos; /* last sequence number (inclusive) */ 338 | struct imcs_iterator_t_* opd[3]; /*operands of sequence operator */ 339 | imcs_elem_typeid_t elem_type; /* result element type */ 340 | uint32 iterator_size; /* size fo iterator + tile data + context */ 341 | imcs_timeseries_t* cs_hdr; /* header of stored timeseries, NULL for sequence iterators */ 342 | void* context; 343 | imcs_tile_t tile; /* tile of values */ 344 | } imcs_iterator_t, *imcs_iterator_h; 345 | 346 | void* imcs_alloc(size_t size); 347 | void imcs_free(void* ptr); 348 | void* imcs_alloc_aligned(size_t size); 349 | uint64 imcs_used_memory(void); 350 | 351 | imcs_timeseries_t* imcs_get_timeseries(char const* id, imcs_elem_typeid_t elem_type, bool is_timestamp, int elem_size, bool create); 352 | imcs_page_t* imcs_new_page(void); 353 | void imcs_free_page(imcs_page_t* pg); 354 | 355 | imcs_iterator_h imcs_new_iterator(size_t elem_size, size_t context_size); 356 | imcs_iterator_h imcs_clone_iterator(imcs_iterator_h iterator); 357 | void imcs_reset_iterator(imcs_iterator_h); 358 | 359 | imcs_iterator_h imcs_cast(imcs_iterator_h input, imcs_elem_typeid_t elem_type, int elem_size); 360 | imcs_count_t imcs_count(imcs_iterator_h input); 361 | 362 | imcs_iterator_h imcs_parallel_iterator(imcs_iterator_h iterator); 363 | 364 | struct imcs_adt_parser_t; 365 | typedef Datum (*imcs_adt_parse_t)(struct imcs_adt_parser_t* parser, char* value, size_t size); 366 | 367 | typedef struct imcs_adt_parser_t { 368 | imcs_adt_parse_t parse; 369 | FmgrInfo proc; 370 | Oid input_oid; 371 | Oid param_oid; 372 | } imcs_adt_parser_t; 373 | 374 | static inline imcs_iterator_h imcs_operand(imcs_iterator_h iterator) 375 | { 376 | return (iterator->opd[0] == NULL) ? imcs_clone_iterator(iterator) : iterator; /* clone leaves */ 377 | } 378 | 379 | 380 | #endif 381 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | # Use -O3 instead of default for PostgreSQL -O2 to enable loop and vectorization optiomizations in GCC. 2 | PATH=/usr/local/pgsql/bin/:$PATH make USE_PGXS=1 CFLAGS="-O3 -Wall -Wno-format-security" LDFLAGS="-pthread" install -------------------------------------------------------------------------------- /smp.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "smp.h" 6 | 7 | #define IMCS_SMP_CHECK(call) if ((call) != 0) imcs_error_handler(__FILE__, __LINE__, #call) 8 | 9 | #ifdef _WIN32 10 | 11 | #include 12 | 13 | #define IMCS_MAX_SEM_VALUE 1000000 14 | 15 | #ifndef TLS_OUT_OF_INDEXES 16 | #define TLS_OUT_OF_INDEXES 0xffffffff 17 | #endif 18 | 19 | static void imcs_error_handler(char const* file, int line, char const* msg) 20 | { 21 | elog(ERROR, "%s:%d %s error=%d", file, line, msg, GetLastError()); 22 | } 23 | 24 | 25 | typedef struct imcs_win_tls_t 26 | { 27 | imcs_tls_t vtab; 28 | unsigned int key; 29 | } imcs_win_tls_t; 30 | 31 | static void* imcs_get_tls(imcs_tls_t* t) 32 | { 33 | imcs_win_tls_t* tls = (imcs_win_tls_t*)t; 34 | return TlsGetValue(tls->key); 35 | } 36 | 37 | static void imcs_set_tls(imcs_tls_t* t, void* value) 38 | { 39 | imcs_win_tls_t* tls = (imcs_win_tls_t*)t; 40 | TlsSetValue(tls->key, value); 41 | } 42 | 43 | static void imcs_destroy_tls(imcs_tls_t* t) 44 | { 45 | imcs_win_tls_t* tls = (imcs_win_tls_t*)t; 46 | TlsFree(tls->key); 47 | free(tls); 48 | } 49 | 50 | 51 | imcs_tls_t* imcs_create_tls(void) 52 | { 53 | imcs_win_tls_t* tls = (imcs_win_tls_t*)malloc(sizeof(imcs_win_tls_t)); 54 | tls->vtab.get = imcs_get_tls; 55 | tls->vtab.set = imcs_set_tls; 56 | tls->vtab.destroy = imcs_destroy_tls; 57 | tls->key = TlsAlloc(); 58 | if (tls->key == TLS_OUT_OF_INDEXES) { 59 | free(tls); 60 | return NULL; 61 | } 62 | return &tls->vtab; 63 | } 64 | 65 | typedef struct imcs_win_thread_t { 66 | imcs_thread_t vtab; 67 | HANDLE handle; 68 | } imcs_win_thread_t; 69 | 70 | static void imcs_join_thread(imcs_thread_t* t) 71 | { 72 | imcs_win_thread_t* thread = (imcs_win_thread_t*)t; 73 | IMCS_SMP_CHECK(WaitForSingleObject(thread->handle, INFINITE) == WAIT_OBJECT_0); 74 | IMCS_SMP_CHECK(CloseHandle(thread->handle)); 75 | free(thread); 76 | } 77 | 78 | imcs_thread_t* imcs_create_thread(imcs_thread_proc_t proc, void* arg) 79 | { 80 | DWORD threadid; 81 | HANDLE h = CreateThread(NULL, 0, (LPTHREAD_START_ROUTINE) proc, arg, 0, &threadid); 82 | if (h == NULL) { 83 | return NULL; 84 | } else { 85 | imcs_win_thread_t* thread = (imcs_win_thread_t*)malloc(sizeof(imcs_win_thread_t)); 86 | thread->vtab.join = imcs_join_thread; 87 | thread->handle = h; 88 | return &thread->vtab; 89 | } 90 | } 91 | 92 | int imcs_get_number_of_cpus(void) 93 | { 94 | SYSTEM_INFO sysinfo; 95 | GetSystemInfo(&sysinfo); 96 | return sysinfo.dwNumberOfProcessors; 97 | } 98 | 99 | typedef struct 100 | { 101 | imcs_mutex_t vtab; 102 | CRITICAL_SECTION cs; 103 | } imcs_win_mutex_t; 104 | 105 | static void imcs_lock_mutex(imcs_mutex_t* m) 106 | { 107 | imcs_win_mutex_t* mutex = (imcs_win_mutex_t*)m; 108 | EnterCriticalSection(&mutex->cs); 109 | } 110 | 111 | static void imcs_unlock_mutex(imcs_mutex_t* m) 112 | { 113 | imcs_win_mutex_t* mutex = (imcs_win_mutex_t*)m; 114 | LeaveCriticalSection(&mutex->cs); 115 | } 116 | 117 | static void imcs_destroy_mutex(imcs_mutex_t* m) 118 | { 119 | imcs_win_mutex_t* mutex = (imcs_win_mutex_t*)m; 120 | DeleteCriticalSection(&mutex->cs); 121 | free(mutex); 122 | } 123 | 124 | 125 | imcs_mutex_t* imcs_create_mutex() 126 | { 127 | imcs_win_mutex_t* mutex = (imcs_win_mutex_t*)malloc(sizeof(imcs_win_mutex_t)); 128 | InitializeCriticalSection(&mutex->cs); 129 | mutex->vtab.lock = imcs_lock_mutex; 130 | mutex->vtab.unlock = imcs_unlock_mutex; 131 | mutex->vtab.destroy = imcs_destroy_mutex; 132 | return &mutex->vtab; 133 | } 134 | 135 | 136 | typedef struct 137 | { 138 | imcs_semaphore_t vtab; 139 | HANDLE handle; 140 | } imcs_win_semaphore_t; 141 | 142 | static imcs_bool imcs_semaphore_wait(imcs_semaphore_t* s, imcs_mutex_t* mutex, int n, unsigned timeout) 143 | { 144 | imcs_win_semaphore_t* sem = (imcs_win_semaphore_t*)s; 145 | int rc; 146 | mutex->unlock(mutex); 147 | while (--n >= 0) { 148 | rc = WaitForSingleObject(sem->handle, timeout); 149 | if (rc == WAIT_TIMEOUT) { 150 | mutex->lock(mutex); 151 | return 0; 152 | } 153 | IMCS_SMP_CHECK(rc == WAIT_OBJECT_0); 154 | } 155 | mutex->lock(mutex); 156 | return 1; 157 | } 158 | 159 | static void imcs_semaphore_signal(imcs_semaphore_t* s, int inc) 160 | { 161 | imcs_win_semaphore_t* sem = (imcs_win_semaphore_t*)s; 162 | IMCS_SMP_CHECK(ReleaseSemaphore(sem->handle, inc, NULL)); 163 | } 164 | 165 | static void imcs_semaphore_destroy(imcs_semaphore_t* s) 166 | { 167 | imcs_win_semaphore_t* sem = (imcs_win_semaphore_t*)s; 168 | CloseHandle(sem->handle); 169 | free(sem); 170 | } 171 | 172 | imcs_semaphore_t* imcs_create_semaphore(int value) 173 | { 174 | imcs_win_semaphore_t* sem; 175 | HANDLE s = CreateSemaphore(NULL, value, IMCS_MAX_SEM_VALUE, NULL); 176 | IMCS_SMP_CHECK(s != NULL); 177 | sem = (imcs_win_semaphore_t*)malloc(sizeof (imcs_win_semaphore_t)); 178 | sem->vtab.wait = imcs_semaphore_wait; 179 | sem->vtab.signal = imcs_semaphore_signal; 180 | sem->vtab.destroy = imcs_semaphore_destroy; 181 | sem->handle = s; 182 | return &sem->vtab; 183 | } 184 | 185 | imcs_process_t imcs_get_pid(void) 186 | { 187 | return (imcs_process_t)(imcs_size_t)GetCurrentProcessId(); 188 | } 189 | 190 | imcs_bool imcs_is_process_alive(imcs_process_t proc) 191 | { 192 | HANDLE h = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, (DWORD)(imcs_size_t)proc); 193 | if (h == NULL) { 194 | return 0; 195 | } 196 | CloseHandle(h); 197 | return 1; 198 | } 199 | 200 | #else 201 | 202 | #include 203 | #include 204 | #include 205 | #include 206 | #include 207 | 208 | static void imcs_error_handler(char const* file, int line, char const* msg) 209 | { 210 | elog(ERROR, "%s:%d %s (%m)", file, line, msg); 211 | } 212 | 213 | /* Posix threads */ 214 | typedef struct imcs_posix_tls_t 215 | { 216 | imcs_tls_t vtab; 217 | pthread_key_t key; 218 | } imcs_posix_tls_t; 219 | 220 | static void* imcs_get_tls(imcs_tls_t* t) 221 | { 222 | imcs_posix_tls_t* tls = (imcs_posix_tls_t*)t; 223 | return pthread_getspecific(tls->key); 224 | } 225 | 226 | static void imcs_set_tls(imcs_tls_t* t, void* value) 227 | { 228 | imcs_posix_tls_t* tls = (imcs_posix_tls_t*)t; 229 | IMCS_SMP_CHECK(pthread_setspecific(tls->key, value)); 230 | } 231 | 232 | static void imcs_destroy_tls(imcs_tls_t* t) 233 | { 234 | imcs_posix_tls_t* tls = (imcs_posix_tls_t*)t; 235 | pthread_key_delete(tls->key); 236 | free(tls); 237 | } 238 | 239 | 240 | imcs_tls_t* imcs_create_tls(void) 241 | { 242 | imcs_posix_tls_t* tls = (imcs_posix_tls_t*)malloc(sizeof(imcs_posix_tls_t)); 243 | tls->vtab.get = imcs_get_tls; 244 | tls->vtab.set = imcs_set_tls; 245 | tls->vtab.destroy = imcs_destroy_tls; 246 | IMCS_SMP_CHECK(pthread_key_create(&tls->key, NULL)); 247 | return &tls->vtab; 248 | } 249 | 250 | typedef struct imcs_posix_thread_t { 251 | imcs_thread_t vtab; 252 | pthread_t thread; 253 | } imcs_posix_thread_t; 254 | 255 | static void imcs_join_thread(imcs_thread_t* t) 256 | { 257 | IMCS_SMP_CHECK(pthread_join(((imcs_posix_thread_t*)t)->thread, NULL)); 258 | free(t); 259 | } 260 | 261 | imcs_thread_t* imcs_create_thread(imcs_thread_proc_t proc, void* arg) 262 | { 263 | imcs_posix_thread_t* t = (imcs_posix_thread_t*)malloc(sizeof(imcs_posix_thread_t)); 264 | IMCS_SMP_CHECK(pthread_create(&t->thread, NULL, (void*(*)(void*))proc, arg)); 265 | t->vtab.join = imcs_join_thread; 266 | return &t->vtab; 267 | } 268 | 269 | #if defined(_SC_NPROCESSORS_ONLN) 270 | int imcs_get_number_of_cpus(void) 271 | { 272 | return sysconf(_SC_NPROCESSORS_ONLN); 273 | } 274 | #elif defined(__linux__) 275 | #include 276 | int imcs_get_number_of_cpus(void) 277 | { 278 | return smp_num_cpus; 279 | } 280 | #elif defined(__FreeBSD__) || defined(__bsdi__) || defined(__OpenBSD__) || defined(__NetBSD__) 281 | #if defined(__bsdi__) || defined(__OpenBSD__) 282 | #include 283 | #endif 284 | #include 285 | int imcs_get_number_of_cpus(void) 286 | { 287 | int mib[2],ncpus=0; 288 | size_t len=sizeof(ncpus); 289 | mib[0]= CTL_HW; 290 | mib[1]= HW_NCPU; 291 | sysctl(mib,2,&ncpus,&len,NULL,0); 292 | return ncpus; 293 | } 294 | #else 295 | int imcs_get_number_of_cpus(void) 296 | { 297 | return 1; 298 | } 299 | #endif 300 | 301 | typedef struct 302 | { 303 | imcs_mutex_t vtab; 304 | pthread_mutex_t cs; 305 | } imcs_posix_mutex_t; 306 | 307 | static void imcs_lock_mutex(imcs_mutex_t* m) 308 | { 309 | imcs_posix_mutex_t* mutex = (imcs_posix_mutex_t*)m; 310 | IMCS_SMP_CHECK(pthread_mutex_lock(&mutex->cs)); 311 | } 312 | 313 | static void imcs_unlock_mutex(imcs_mutex_t* m) 314 | { 315 | imcs_posix_mutex_t* mutex = (imcs_posix_mutex_t*)m; 316 | IMCS_SMP_CHECK(pthread_mutex_unlock(&mutex->cs)); 317 | } 318 | 319 | static void imcs_destroy_mutex(imcs_mutex_t* m) 320 | { 321 | imcs_posix_mutex_t* mutex = (imcs_posix_mutex_t*)m; 322 | pthread_mutex_destroy(&mutex->cs); 323 | free(mutex); 324 | } 325 | 326 | 327 | imcs_mutex_t* imcs_create_mutex(void) 328 | { 329 | imcs_posix_mutex_t* mutex = (imcs_posix_mutex_t*)malloc(sizeof(imcs_posix_mutex_t)); 330 | IMCS_SMP_CHECK(pthread_mutex_init(&mutex->cs, NULL)); 331 | mutex->vtab.lock = imcs_lock_mutex; 332 | mutex->vtab.unlock = imcs_unlock_mutex; 333 | mutex->vtab.destroy = imcs_destroy_mutex; 334 | return &mutex->vtab; 335 | } 336 | 337 | 338 | typedef struct 339 | { 340 | imcs_semaphore_t vtab; 341 | pthread_cond_t cond; 342 | int count; 343 | } imcs_posix_semaphore_t; 344 | 345 | static imcs_bool imcs_semaphore_wait(imcs_semaphore_t* s, imcs_mutex_t* mutex, int n, unsigned timeout) 346 | { 347 | imcs_posix_semaphore_t* sem = (imcs_posix_semaphore_t*)s; 348 | struct timespec abs_ts; 349 | int rc; 350 | 351 | if (timeout != IMCS_TM_INFINITE) { 352 | #ifdef PTHREAD_GET_EXPIRATION_NP 353 | struct timespec rel_ts; 354 | rel_ts.tv_sec = timeout/1000; 355 | rel_ts.tv_nsec = timeout%1000*1000000; 356 | pthread_get_expiration_np(&rel_ts, &abs_ts); 357 | #else 358 | struct timeval cur_tv; 359 | gettimeofday(&cur_tv, NULL); 360 | abs_ts.tv_sec = cur_tv.tv_sec + timeout/1000; 361 | abs_ts.tv_nsec = cur_tv.tv_usec*1000 + timeout%1000*1000000; 362 | if (abs_ts.tv_nsec > 1000000000) { 363 | abs_ts.tv_nsec -= 1000000000; 364 | abs_ts.tv_sec += 1; 365 | } 366 | #endif 367 | } 368 | while (sem->count < n) 369 | { 370 | rc = timeout != IMCS_TM_INFINITE 371 | ? pthread_cond_timedwait(&sem->cond, &((imcs_posix_mutex_t*)mutex)->cs, &abs_ts) 372 | : pthread_cond_wait(&sem->cond, &((imcs_posix_mutex_t*)mutex)->cs); 373 | if (rc == ETIMEDOUT) { 374 | return 0; 375 | } 376 | IMCS_SMP_CHECK(rc); 377 | } 378 | sem->count -= n; 379 | return 1; 380 | } 381 | 382 | static void imcs_semaphore_signal(imcs_semaphore_t* s, int inc) 383 | { 384 | imcs_posix_semaphore_t* sem = (imcs_posix_semaphore_t*)s; 385 | sem->count += inc; 386 | if (inc > 1) 387 | { 388 | IMCS_SMP_CHECK(pthread_cond_broadcast(&sem->cond)); 389 | } 390 | else if (inc == 1) 391 | { 392 | IMCS_SMP_CHECK(pthread_cond_signal(&sem->cond)); 393 | } 394 | } 395 | 396 | static void imcs_semaphore_destroy(imcs_semaphore_t* s) 397 | { 398 | imcs_posix_semaphore_t* sem = (imcs_posix_semaphore_t*)s; 399 | pthread_cond_destroy(&sem->cond); 400 | free(sem); 401 | } 402 | 403 | imcs_semaphore_t* imcs_create_semaphore(int value) 404 | { 405 | imcs_posix_semaphore_t* sem = (imcs_posix_semaphore_t*)malloc(sizeof(imcs_posix_semaphore_t)); 406 | IMCS_SMP_CHECK(pthread_cond_init(&sem->cond, NULL)); 407 | sem->vtab.wait = imcs_semaphore_wait; 408 | sem->vtab.signal = imcs_semaphore_signal; 409 | sem->vtab.destroy = imcs_semaphore_destroy; 410 | sem->count = value; 411 | return &sem->vtab; 412 | } 413 | 414 | imcs_process_t imcs_get_pid(void) 415 | { 416 | return (imcs_process_t)(size_t)getpid(); 417 | } 418 | 419 | imcs_bool imcs_is_process_alive(imcs_process_t proc) 420 | { 421 | return kill((int)(size_t)proc, 0) == 0; 422 | } 423 | 424 | #endif 425 | -------------------------------------------------------------------------------- /smp.h: -------------------------------------------------------------------------------- 1 | #ifndef __SMP_H__ 2 | #define __SMP_H__ 3 | 4 | typedef void* imcs_process_t; 5 | typedef int imcs_bool; 6 | 7 | #define IMCS_TM_INFINITE ((unsigned)-1) 8 | 9 | typedef void (*imcs_thread_proc_t)(void* arg); 10 | 11 | typedef struct imcs_tls_t { 12 | void* (*get)(struct imcs_tls_t* tls); 13 | void (*set)(struct imcs_tls_t* tls, void* value); 14 | void (*destroy)(struct imcs_tls_t* tls); 15 | } imcs_tls_t; 16 | 17 | imcs_tls_t* imcs_create_tls(void); 18 | 19 | 20 | typedef struct imcs_thread_t { 21 | void (*join)(struct imcs_thread_t* id); 22 | } imcs_thread_t; 23 | 24 | extern imcs_thread_t* imcs_create_thread(imcs_thread_proc_t proc, void* arg); 25 | 26 | typedef struct imcs_mutex_t { 27 | void (*lock)(struct imcs_mutex_t* m); 28 | void (*unlock)(struct imcs_mutex_t* m); 29 | void (*destroy)(struct imcs_mutex_t* sem); 30 | } imcs_mutex_t; 31 | 32 | extern imcs_mutex_t* imcs_create_mutex(void); 33 | 34 | typedef struct imcs_semaphore_t { 35 | imcs_bool (*wait)(struct imcs_semaphore_t* sem, imcs_mutex_t* mutex, int n, unsigned timeout); 36 | void (*signal)(struct imcs_semaphore_t* sem, int n); 37 | void (*destroy)(struct imcs_semaphore_t* sem); 38 | } imcs_semaphore_t; 39 | 40 | extern imcs_semaphore_t* imcs_create_semaphore(int value); 41 | 42 | extern int imcs_get_number_of_cpus(void); 43 | extern imcs_process_t imcs_get_pid(void); 44 | extern imcs_bool imcs_is_process_alive(imcs_process_t proc); 45 | 46 | 47 | typedef void (*imcs_job_t)(int thread_id, int n_threads, void* arg); 48 | typedef void (*imcs_job_callback_t)(void* arg, void* result); 49 | 50 | typedef struct imcs_thread_pool_t { 51 | int (*get_number_of_threads)(struct imcs_thread_pool_t* pool); 52 | void (*execute)(struct imcs_thread_pool_t* pool, imcs_job_t job, void* arg); 53 | void (*merge)(struct imcs_thread_pool_t* pool, imcs_job_callback_t callback, void* result); 54 | void (*destroy)(struct imcs_thread_pool_t* pool); 55 | } imcs_thread_pool_t; 56 | 57 | extern imcs_thread_pool_t* imcs_create_thread_pool(int n_threads); /* 0 - choose number of threads automaticallym based on number of cores */ 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /sql/create.sql: -------------------------------------------------------------------------------- 1 | create table Quote (Symbol char(10), Day date, Open real, High real, Low real, Close real, Volume integer); 2 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('01-Nov-2013'), 10.2, 11.0, 10.0, 10.5, 100); 3 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('02-Nov-2013'), 20.2, 20.2, 20.2, 20.2, 200); 4 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('04-Nov-2013'), 30.5, 31.0, 30.0, 30.2, 300); 5 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('05-Nov-2013'), 40.5, 41.0, 40.0, 40.2, 400); 6 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('IBM', date('06-Nov-2013'), 50.2, 51.0, 50.0, 50.5, 500); 7 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('ABB', date('03-Nov-2013'), 60.5, 61.0, 70.0, 60.2, 600); 8 | insert into Quote (Symbol,Day,Open,High,Low,Close,Volume) values ('ABB', date('06-Nov-2013'), 70.2, 71.0, 70.0, 70.5, 700); 9 | create extension imcs; 10 | select cs_create('Quote', 'Day', 'Symbol'); 11 | select Quote_load(); 12 | 13 | -- Table with varying size strings which are mapped to integers using dictionary 14 | create table CrashLog( 15 | log_time timestamp, ---when this data is produce,selective 16 | uid bigint, ---user id 17 | country varchar,----USA,Chine,Japan etc...cardinal number is ~ 100,has hot spot 18 | city varchar, ---cardinal number~10000,has hot spot 19 | device varchar, ---iphone4,nokiaX etc, cardinal number is 1000,has hot spot 20 | net varchar, -----wifi,Vodafone 3G... cardinal number is 10,hash hot spot 21 | os varchar); ----iOS6.x,Android 2.3 cardinal number is 100,hash hot spot 22 | 23 | insert into CrashLog values ('2014-04-14 11:54', 10000001, 'USA', 'New York', 'iPhone4', 'wifi', 'iOS6.x'); 24 | insert into CrashLog values ('2014-04-14 11:55', 10000002, 'Japan', 'Tokio', 'Sony Xperia Z1', 'Vodafone 3G', 'Android 4.4'); 25 | insert into CrashLog values ('2014-04-14 11:56', 10000003, 'China', 'Beijing', 'iPhone5', 'wifi', 'iOS7.x'); 26 | select cs_create('CrashLog', 'log_time'); 27 | select CrashLog_load(); 28 | 29 | select cs_used_memory(); 30 | -------------------------------------------------------------------------------- /sql/cumagg.sql: -------------------------------------------------------------------------------- 1 | select cs_cum_max(Close) from Quote_get('IBM'); 2 | select cs_cum_min(Close) from Quote_get('IBM'); 3 | select cs_cum_sum(Close) from Quote_get('IBM'); 4 | select cs_cum_avg(Close) from Quote_get('IBM'); 5 | select cs_cum_prd(Close) from Quote_get('IBM'); 6 | select cs_cum_var(Close) from Quote_get('IBM'); 7 | select cs_cum_dev(Close) from Quote_get('IBM'); 8 | -------------------------------------------------------------------------------- /sql/datetime.sql: -------------------------------------------------------------------------------- 1 | select cs_project(q.*) from (select cs_cast(Day, 'timestamp') from Quote_get('ABB')) q; 2 | select cs_year(Day) from Quote_get('ABB'); 3 | select cs_month(Day) from Quote_get('ABB'); 4 | select cs_mday(Day) from Quote_get('ABB'); 5 | select cs_wday(Day) from Quote_get('ABB'); 6 | select cs_week(Day) from Quote_get('ABB'); 7 | select cs_quarter(Day) from Quote_get('ABB'); 8 | select cs_year(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 9 | select cs_month(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 10 | select cs_mday(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 11 | select cs_wday(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 12 | select cs_week(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 13 | select cs_quarter(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 14 | select cs_hour(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 15 | select cs_minute(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 16 | select cs_second(cs_cast(Day, 'timestamp')) from Quote_get('ABB'); 17 | select cs_cast(cs_const(date '2013-10-11', 'date'), 'timestamp'); 18 | select cs_const(timestamp '2013-10-11 10:00', 'timestamp'); 19 | select cs_cast(cs_const(timestamp '2013-10-11 10:00', 'timestamp'), 'date'); 20 | -------------------------------------------------------------------------------- /sql/drop.sql: -------------------------------------------------------------------------------- 1 | select Quote_delete('IBM', date('02-Nov-2013')); 2 | select Quote_count('IBM'); 3 | select Day from Quote_get('IBM'); 4 | 5 | select Quote_delete('ABB', date('03-Nov-2013'), date('06-Nov-2013')); 6 | select Quote_truncate(); 7 | 8 | select Quote_count('IBM'); 9 | select Quote_count('ABB'); 10 | select * from Quote_get('IBM'); 11 | select * from Quote_get('ABB'); 12 | 13 | select CrashLog_delete('2014-04-14 11:54', '2014-04-14 11:56'); 14 | 15 | select Quote_drop(); 16 | select CrashLog_drop(); -------------------------------------------------------------------------------- /sql/grandagg.sql: -------------------------------------------------------------------------------- 1 | select Quote_count('IBM'); 2 | select Quote_first('IBM'); 3 | select Quote_last('IBM'); 4 | select cs_sum(Volume) from Quote_get('IBM'); 5 | select cs_max(Close) from Quote_get(array['ABB','IBM'], date('03-Nov-2013'), date('05-Nov-2013')); 6 | select cs_min(Close) from Quote_get(array['ABB','IBM']); 7 | select cs_count(Day) from Quote_get('IBM'); 8 | select cs_approxdc(Volume/200) from Quote_get('IBM'); 9 | select cs_avg(Volume) from Quote_get('IBM'); 10 | select cs_prd(High - Low) from Quote_get('IBM'); 11 | select cs_var(Open) from Quote_get('IBM'); 12 | select cs_dev(Close) from Quote_get('IBM'); 13 | select cs_sum('int4:{1,2,3}'); 14 | select cs_sum(Close) from Quote_concat(array['ABB','IBM']); 15 | select cs_sum(cs_limit(cs_filter_pos(cs_const(1, 'char')), 0, 3)); 16 | select cs_sum(cs_cum_sum(cs_limit(cs_const(1, 'int4'), 0, 3))); 17 | select cs_median(Close) from Quote_get(array['ABB','IBM']); 18 | select cs_all('int2:{2,3,6}'); 19 | select cs_any('char:{2,3,6}'); -------------------------------------------------------------------------------- /sql/gridagg.sql: -------------------------------------------------------------------------------- 1 | select cs_grid_max(Close,2) from Quote_get('IBM'); 2 | select cs_grid_min(Close,2) from Quote_get('IBM'); 3 | select cs_grid_sum(Close,2) from Quote_get('IBM'); 4 | select cs_grid_avg(Close,2) from Quote_get('IBM'); 5 | select cs_grid_var(Close,2) from Quote_get('IBM'); 6 | select cs_grid_dev(Close,2) from Quote_get('IBM'); 7 | -------------------------------------------------------------------------------- /sql/groupbyagg.sql: -------------------------------------------------------------------------------- 1 | select cs_group_max(Close,Day/3) from Quote_get('IBM'); 2 | select cs_group_min(Close,Day/3) from Quote_get('IBM'); 3 | select cs_group_sum(Close,Day/3) from Quote_get('IBM'); 4 | select cs_group_avg(Close,Day/3) from Quote_get('IBM'); 5 | select cs_group_var(Close,Day/3) from Quote_get('IBM'); 6 | select cs_group_dev(Close,Day/3) from Quote_get('IBM'); 7 | select cs_group_first(Close,Day/3) from Quote_get('IBM'); 8 | select cs_group_last(Close,Day/3) from Quote_get('IBM'); 9 | select cs_group_all('int8:{3,1,6,7,0,3,6,5,2,3,7}','int4:{1,1,1,2,2,3,3,4,5,5,5}'); 10 | select cs_group_any('int2:{3,1,6,7,0,3,6,5,2,3,7}','char:{1,1,1,2,2,3,3,4,5,5,5}'); 11 | select cs_win_group_max(Close,cs_week(Day)) from Quote_get('IBM'); 12 | select cs_win_group_min(Close,cs_week(Day)) from Quote_get('IBM'); 13 | select cs_win_group_sum(Close,cs_week(Day)) from Quote_get('IBM'); 14 | select cs_win_group_avg(Close,cs_week(Day)) from Quote_get('IBM'); 15 | select cs_win_group_var(Close,cs_week(Day)) from Quote_get('IBM'); 16 | select cs_win_group_dev(Close,cs_week(Day)) from Quote_get('IBM'); 17 | select cs_win_group_first(Close,cs_week(Day)) from Quote_get('IBM'); 18 | select cs_win_group_last(Close,cs_week(Day)) from Quote_get('IBM'); 19 | select cs_win_group_sum('int4:{1,2,3,4,5,6,7,8,9,10}','int4:{1,1,1,2,2,3,3,3,3,4}'); 20 | -------------------------------------------------------------------------------- /sql/hashagg.sql: -------------------------------------------------------------------------------- 1 | select cs_hash_max(Close,Day % 2) from Quote_get('IBM'); 2 | select cs_hash_min(Close,Day % 2) from Quote_get('IBM'); 3 | select cs_hash_sum(Close,Day % 2) from Quote_get('IBM'); 4 | select cs_hash_avg(Close,Day % 2) from Quote_get('IBM'); 5 | select (q.p).agg_val,cs_cut((q.p).group_by,'i4i4') from (select cs_project_agg(cs_hash_sum(Close,(Day % 2)||(Volume%10))) p from Quote_get('IBM')) q; 6 | select p.agg_val,cs_cut(p.group_by,'i4i4') from (select (cs_project_agg(cs_hash_sum(Close,(Day % 2)||(Volume%10)))).* from Quote_get('IBM')) p; 7 | create type PairOfInt as (first integer, second integer); 8 | select p.agg_val,cs_as(p.group_by,'PairOfInt') from (select (cs_project_agg(cs_hash_sum(Close,(Day % 2)||(Volume%10)))).* from Quote_get('IBM')) p; 9 | 10 | select cs_hash_count(cs_floor((High-Low)*10)) from Quote_get('IBM'); 11 | select cs_hash_dup_count(cs_ceil((High-Low)*10), Day%3) from Quote_get('IBM'); 12 | select cs_hash_all('int4:{3,1,6,7,0,3,6,5,2,3,7}','int8:{1,1,1,2,2,3,3,4,5,5,5}'); 13 | select cs_hash_any('char:{3,1,6,7,0,3,6,5,2,3,7}','int2:{1,1,1,2,2,3,3,4,5,5,5}'); 14 | 15 | select cs_code2str(group_by, 1) as country, cs_code2str(group_by, 2) as device, agg_val as counter from (select (cs_project_agg(cs_hash_count(country||device))).* from CrashLog_get()) s; 16 | 17 | -------------------------------------------------------------------------------- /sql/math.sql: -------------------------------------------------------------------------------- 1 | select cs_sqrt((Open - Close) ^ 2.0) from Quote_get('IBM'); 2 | select cs_sin(Open)*cs_sin(Open) + cs_cos(Open)*cs_cos(Open) from Quote_get('IBM'); 3 | select cs_atan(cs_tan(Close)) from Quote_get('IBM'); 4 | select cs_asin(cs_sin(Close)) from Quote_get('IBM'); 5 | select cs_acos(cs_cos(Close)) from Quote_get('IBM'); 6 | select cs_log(cs_exp(Close/Open)) - Open/Close from Quote_get('IBM'); 7 | select cs_isnan(cs_parse('{-1,0,1}','float8')/0.0); -------------------------------------------------------------------------------- /sql/operators.sql: -------------------------------------------------------------------------------- 1 | select Open+Close from Quote_get('IBM'); 2 | select Open-Close from Quote_get('IBM'); 3 | select Open*Close from Quote_get('IBM'); 4 | select Open/Close from Quote_get('IBM'); 5 | select Open%Close from Quote_get('IBM'); 6 | select Open^Close from Quote_get('IBM'); 7 | select Close+*Volume from Quote_get('IBM'); 8 | select Volume//Close from Quote_get('IBM'); 9 | select Open~Close from Quote_get('IBM'); 10 | select (Open>Close)?Day from Quote_get('IBM'); 11 | select ?(Open>Close) from Quote_get('IBM'); 12 | select -Open from Quote_get('IBM'); 13 | select Open>Close from Quote_get('IBM'); 14 | select Open>=Close from Quote_get('IBM'); 15 | select OpenClose from Quote_get('IBM'); 19 | select (Open>=Close) & (High<>Low) from Quote_get('IBM'); 20 | select (Open>Close) | (High=Low) from Quote_get('IBM'); 21 | select (Open=Close) # (High=Low) from Quote_get('IBM'); 22 | select !(Open=Close) from Quote_get('IBM'); 23 | select ~Volume from Quote_get('IBM'); 24 | select Volume & 1 from Quote_get('IBM'); 25 | select Volume | 1 from Quote_get('IBM'); 26 | select Volume # -1 from Quote_get('IBM'); 27 | select @(Open-Close) from Quote_get('IBM'); 28 | select Day,Open||Close from Quote_get('IBM', date('01-Nov-2013'), date('03-Nov-2013')); 29 | select cs_maxof(Open,Close) from Quote_get('IBM'); 30 | select cs_minof(Open,Close) from Quote_get('IBM'); 31 | select cs_norm(Close) from Quote_get('IBM'); 32 | select Close*'{2.0,2.1,2.2}'::text from Quote_get('IBM', '1-Nov-2013', '4-Nov-2013'); 33 | select Close*'float4:{2.0,2.1,2.2}' from Quote_get('IBM', '1-Nov-2013', '4-Nov-2013'); 34 | select Day=date('1-Nov-2013') from Quote_get('IBM'); 35 | select Volume*2 from Quote_get('ABB'); 36 | select cs_parse('{1,2,3,4}','int4')+cs_const(3.14,'float4'); 37 | select cs_parse('{1,2,3,4}','int4')+10; 38 | select cs_parse('{1,2,3,4}', 'int4')+'{10,20,30,40}'::text; 39 | select cs_parse('{11-Nov-2013,12-Nov-2013,30-Nov-2013}', 'date'); 40 | select cs_concat('int4:{1,2,3}','int4:{4,5,6}'); 41 | select cs_parse('{1,2,3,4,5}', 'int4') << 1; 42 | select cs_parse('{1,2,3,4,5}', 'int4') >> 2; 43 | select cs_parse('{1,2,3,4,5}', 'int4') << 10; 44 | select cs_parse('{1,2,3,4,5}', 'int4') >> 10; 45 | select cs_like('bpchar8:{Abc,aaBcc,aabbcc,abcba,aabbabcc,aabac,bca}', '%abc%'); 46 | select cs_ilike('bpchar8:{Abc,aaBcc,aabbcc,abcba,aabbabcc,aabac,bca}', '%abc%'); 47 | select cs_parse('{Abc,aaBcc,aabbcc,abcba,aabbabcc,aabac,bca}', 'bpchar', 8) ~~ '%abc%'; 48 | select * from CrashLog_get(); 49 | select cs_project(c.*, cs_filter_pos(country = cs_str2code('USA'))) from CrashLog_get() c; 50 | select cs_project(c.*, cs_filter_pos(cs_ilike(device, 'iphone%'))) from CrashLog_get() c; 51 | select cs_dictionary_size(); 52 | -------------------------------------------------------------------------------- /sql/scalarop.sql: -------------------------------------------------------------------------------- 1 | select cs_wsum(Volume,Close) from Quote_get('IBM'); 2 | select cs_wavg(Volume,Close) from Quote_get('IBM'); 3 | select cs_corr(High,Low) from Quote_get('IBM'); 4 | select cs_cov(High,Low) from Quote_get('IBM'); 5 | -------------------------------------------------------------------------------- /sql/sort.sql: -------------------------------------------------------------------------------- 1 | select cs_top_max(Close, 3) from Quote_get('IBM'); 2 | select cs_top_min(Close, 3) from Quote_get('IBM'); 3 | select cs_top_max_pos(Close, 3) from Quote_get('IBM'); 4 | select cs_top_min_pos(Close, 3) from Quote_get('IBM'); 5 | select Quote_project(q.*, cs_top_max_pos(Close, 3)) from Quote_get('IBM') q; 6 | select cs_sort(Close) from Quote_get('IBM'); 7 | select cs_sort(Close, 'desc') from Quote_get('IBM'); 8 | select Quote_project(q.*, cs_sort_pos(Close)) from Quote_get('IBM') q; 9 | select Quote_project(q.*, cs_sort_pos(Close, 'desc')) from Quote_get('IBM') q; 10 | select cs_rank(Close) from Quote_get('IBM'); 11 | select cs_rank('float4:{1.1,0.1,2.2,0.2,0.1}'); 12 | select cs_dense_rank('float4:{1.1,0.1,2.2,0.2,0.1}'); 13 | select cs_rank('float4:{1.1,0.2,2.2,0.2,0.1}', 'desc'); 14 | select cs_dense_rank('float4:{1.1,0.2,2.2,0.2,0.1}', 'desc'); 15 | select cs_quantile(Close, 2) from Quote_get('IBM'); 16 | select cs_quantile('float4:{10,3,0,3,4,5,9,11,7,3,3}', 2); 17 | -------------------------------------------------------------------------------- /sql/span.sql: -------------------------------------------------------------------------------- 1 | select Close from Quote_get('IBM', '2-Nov-2013', '5-Nov-2013'); 2 | select Close from Quote_get('IBM', '2-Nov-2013'); 3 | select Close from Quote_get('IBM', from_ts:='2-Nov-2013'); 4 | select Close from Quote_get('IBM', till_ts:='5-Nov-2013'); 5 | select Close from Quote_get('IBM'); 6 | 7 | select Close from Quote_span('IBM', 1, 3); 8 | select Close from Quote_span('IBM', 1); 9 | select Close from Quote_span('IBM', from_pos:=1); 10 | select Close from Quote_span('IBM', till_pos:=3); 11 | select Close from Quote_span('IBM'); 12 | 13 | select Close from Quote_get(array['ABB','IBM'], '2-Nov-2013', '5-Nov-2013'); 14 | select Close from Quote_get(array['ABB','IBM'], '2-Nov-2013'); 15 | select Close from Quote_get(array['ABB','IBM'], from_ts:='2-Nov-2013'); 16 | select Close from Quote_get(array['ABB','IBM'], till_ts:='5-Nov-2013'); 17 | select Close from Quote_get(array['ABB','IBM']); 18 | 19 | select Close from Quote_span(array['ABB','IBM'], 1, 3); 20 | select Close from Quote_span(array['ABB','IBM'], 1); 21 | select Close from Quote_span(array['ABB','IBM'], from_pos:=1); 22 | select Close from Quote_span(array['ABB','IBM'], till_pos:=3); 23 | select Close from Quote_span(array['ABB','IBM']); -------------------------------------------------------------------------------- /sql/spec.sql: -------------------------------------------------------------------------------- 1 | select cs_histogram(Close, 0, 100, 10) from Quote_get('IBM'); 2 | select cs_cross(Open-Close, 0) from Quote_get('IBM'); 3 | select cs_extrema(cs_parse('{1,2,3,2,1,0,0,1,1,2,4,0}','int4'), 0); 4 | select cs_extrema(cs_parse('{1,2,3,2,1,0,0,1,1,2,4,0}','int4'), 1); 5 | select cs_extrema(cs_parse('{1,2,3,2,1,0,0,1,1,2,4,0}','int4'), -1); 6 | select cs_stretch(cs_parse('{1,2,3,4,5}','int4'), cs_parse('{2,4}','int4'), cs_parse('{1.1,2.2}','float8'), 1.0); 7 | select cs_stretch0(cs_parse('{1,2,3,5}','int8'), cs_parse('{2,3,4}','int8'), cs_parse('{1.1,1.2,1.3}','float4'), 0.0); 8 | select cs_asof_join(cs_parse('{4,9}','int4'), cs_parse('{1,3,6,10}','int4'), cs_parse('{0.1,0.3,0.6,1.0}','float8')); 9 | select cs_asof_join_pos('int8:{4,9}', 'int8:{1,3,6,10}'); 10 | select (Quote_project(q.*,'date:{31-Oct-2013,03-Nov-2013}'->Day)).* from Quote_get('IBM') q; 11 | select (Quote_project(q.*,'date:{1-Nov-2013,2-Nov-2013,03-Nov-2013,04-Nov-2013}'<->Day)).* from Quote_get('IBM') q; 12 | select cs_join('date:{1-Nov-2013,2-Nov-2013,03-Nov-2013,04-Nov-2013}', Day, Close) from Quote_get('IBM'); 13 | select (Quote_project(Quote_get('IBM'), Quote_join('IBM', 'date:{2-Nov-2013,4-Nov-2013,03-Nov-2013,01-Nov-2013}'))).*; 14 | select (Quote_project(Quote_get('IBM'), Quote_join('IBM', 'date:{2-Nov-2013,4-Nov-2013,03-Nov-2013,01-Nov-2013}', 1))).*; 15 | select (Quote_project(Quote_get('IBM'), Quote_join('IBM', 'date:{2-Nov-2013,4-Nov-2013,03-Nov-2013,01-Nov-2013}', 0))).*; 16 | select (Quote_project(Quote_get('IBM'), Quote_join('IBM', 'date:{2-Nov-2013,4-Nov-2013,03-Nov-2013,01-Nov-2013}', -1))).*; -------------------------------------------------------------------------------- /sql/transform.sql: -------------------------------------------------------------------------------- 1 | select cs_cast(Volume, 'float4') from Quote_get('IBM'); 2 | select cs_iif(Volume>10, Open, Close) from Quote_get('IBM'); 3 | select cs_if(Open Close, Day) from Quote_get('IBM'); 9 | select Quote_project(q.*, cs_filter_pos(High > Low*1.01)) from Quote_get('IBM', null, date('05-Nov-2013')) q; 10 | select Quote_project(q.*, cs_filter_first_pos(High > Low*1.01, 3)) from Quote_get('IBM') q; 11 | select cs_unique(Volume/200) from Quote_get('IBM'); 12 | select cs_reverse(Day) from Quote_get('IBM'); 13 | select cs_diff(Close) from Quote_get('IBM'); 14 | select cs_trend(Close) from Quote_get('IBM'); 15 | select cs_project(q.*) from (select Day,cs_maxof(Open,Close) from Quote_get('IBM', date('02-Nov-2013'))) q; 16 | select cs_map(Volume, cs_top_max_pos(Close, 1)) from Quote_get('IBM'); 17 | select cs_union(ibm.Day, abb.Day) from Quote_get('ABB') as abb, Quote_get('IBM') as ibm; 18 | create function mul2(x float) returns float as $$ begin return x*2; end; $$ language plpgsql strict immutable; 19 | select cs_call(Close, 'mul2'::regproc) from Quote_get('IBM'); 20 | select cs_call(Close, 'sin'::regproc) from Quote_get('IBM'); 21 | select cs_to_int4_array('int4:{1,2,3,4,5}'); 22 | select cs_to_float4_array('int4:{1,2,3,4,5}'); 23 | select cs_sum(cs_from_array(array[1,2,3,4,5])); 24 | select cs_to_bpchar_array('bpchar10:{Hello,World,!}'); 25 | select cs_from_array(array['Hello','World','!'], 10); 26 | select cs_parse('{100.99,99.01,"$1,000,000"}', 'money'); 27 | select cs_cast(cs_parse('{100.99,99.01,1000000}', 'money') * 2, 'money'); 28 | select cs_cast('float4:{100.99,99.01,1000000}', 'money'); 29 | select cs_cast('money:{100.99,99.01,"1,000,000"}', 'float8'); 30 | select cs_trend('int4:{1,2,3,3,2,2,4,5,6,5,5}'); 31 | 32 | -------------------------------------------------------------------------------- /sql/windowagg.sql: -------------------------------------------------------------------------------- 1 | select cs_window_max(Close,3) from Quote_get('IBM'); 2 | select cs_window_min(Close,3) from Quote_get('IBM'); 3 | select cs_window_sum(Close,3) from Quote_get('IBM'); 4 | select cs_window_avg(Close,3) from Quote_get('IBM'); 5 | select cs_window_var(Close,3) from Quote_get('IBM'); 6 | select cs_window_dev(Close,3) from Quote_get('IBM'); 7 | --- Leave only elements corresponding to entire window 8 | select cs_limit(cs_window_max(Close,3), 2) from Quote_get('IBM'); 9 | select cs_window_min(Close,3) << 2 from Quote_get('IBM'); 10 | select cs_window_sum(Close,3) << 2 from Quote_get('IBM'); 11 | select cs_window_avg(Close,3) << 2 from Quote_get('IBM'); 12 | select cs_window_var(Close,3) << 2 from Quote_get('IBM'); 13 | select cs_window_dev(Close,3) << 2 from Quote_get('IBM'); 14 | --- 2-days EMA (Exponential Moving Average) 15 | select cs_window_ema(Close,3) from Quote_get('IBM'); 16 | --- 2-days ATR (Average True Range) 17 | select cs_window_atr(cs_maxof(High-Low,cs_concat('float4:{0}',cs_maxof(cs_abs((High<<1) - Close), cs_abs((Low<<1) - Close)))), 3) << 2 from Quote_get('IBM'); 18 | -------------------------------------------------------------------------------- /sysv_shmem.patch: -------------------------------------------------------------------------------- 1 | --- src/backend/port/sysv_shmem.c.orig 2013-11-19 16:45:58.825487864 +0400 2 | +++ src/backend/port/sysv_shmem.c 2013-11-19 16:45:44.485488312 +0400 3 | @@ -61,6 +61,8 @@ 4 | #define MAP_FAILED ((void *) -1) 5 | #endif 6 | 7 | +#define LINUX_SHMEM_LIMIT (256LL*1024*1024*1024) 8 | + 9 | 10 | unsigned long UsedShmemSegID = 0; 11 | void *UsedShmemSegAddr = NULL; 12 | @@ -369,10 +371,17 @@ 13 | IpcMemoryId shmid; 14 | struct stat statbuf; 15 | Size sysvsize = size; 16 | + int mmapFlags = PG_MMAP_FLAGS; 17 | 18 | /* Room for a header? */ 19 | Assert(size > MAXALIGN(sizeof(PGShmemHeader))); 20 | 21 | +#ifdef MAP_HUGETLB 22 | + if (size > LINUX_SHMEM_LIMIT) { 23 | + mmapFlags |= MAP_HUGETLB; 24 | + } 25 | +#endif 26 | + 27 | /* 28 | * As of PostgreSQL 9.3, we normally allocate only a very small amount of 29 | * System V shared memory, and only for the purposes of providing an 30 | @@ -410,8 +419,7 @@ 31 | * out to be false, we might need to add a run-time test here and do 32 | * this only if the running kernel supports it. 33 | */ 34 | - AnonymousShmem = mmap(NULL, size, PROT_READ | PROT_WRITE, PG_MMAP_FLAGS, 35 | - -1, 0); 36 | + AnonymousShmem = mmap(NULL, size, PROT_READ | PROT_WRITE, mmapFlags, -1, 0); 37 | if (AnonymousShmem == MAP_FAILED) 38 | ereport(FATAL, 39 | (errmsg("could not map anonymous shared memory: %m"), 40 | -------------------------------------------------------------------------------- /threadpool.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include "smp.h" 3 | 4 | typedef struct imcs_thread_pool_impl_t 5 | { 6 | imcs_thread_pool_t vtab; 7 | int n_workers; 8 | imcs_thread_t** workers; 9 | void* arg; 10 | imcs_job_t job; 11 | int work_id; 12 | imcs_mutex_t* mutex; 13 | imcs_mutex_t* sync; 14 | imcs_semaphore_t* start; 15 | imcs_semaphore_t* finish; 16 | int stop; 17 | } imcs_thread_pool_impl_t; 18 | 19 | 20 | static void imcs_thread_pool_worker(imcs_thread_pool_impl_t* pool) 21 | { 22 | int work_id; 23 | pool->sync->lock(pool->sync); 24 | while (1) { 25 | pool->start->wait(pool->start, pool->sync, 1, IMCS_TM_INFINITE); 26 | if (pool->stop) { 27 | pool->finish->signal(pool->finish, 1); 28 | pool->sync->unlock(pool->sync); 29 | break; 30 | } 31 | work_id = pool->work_id++; 32 | pool->sync->unlock(pool->sync); 33 | pool->job(work_id, pool->n_workers, pool->arg); 34 | pool->sync->lock(pool->sync); 35 | pool->finish->signal(pool->finish, 1); 36 | } 37 | } 38 | 39 | static void imcs_thread_pool_wait(imcs_thread_pool_impl_t* pool) 40 | { 41 | pool->sync->lock(pool->sync); 42 | pool->work_id = 0; 43 | pool->start->signal(pool->start, pool->n_workers); 44 | pool->finish->wait(pool->finish, pool->sync, pool->n_workers, IMCS_TM_INFINITE); 45 | pool->sync->unlock(pool->sync); 46 | 47 | } 48 | int counters[4] = {0,0,0,0}; 49 | static void imcs_thread_pool_execute(struct imcs_thread_pool_t* self, imcs_job_t job, void* arg) 50 | { 51 | imcs_thread_pool_impl_t* pool = (imcs_thread_pool_impl_t*)self; 52 | pool->mutex->lock(pool->mutex); 53 | pool->job = job; 54 | pool->arg = arg; 55 | imcs_thread_pool_wait(pool); 56 | pool->mutex->unlock(pool->mutex); 57 | } 58 | 59 | static void imcs_thread_pool_destroy(struct imcs_thread_pool_t* self) 60 | { 61 | int i; 62 | imcs_thread_pool_impl_t* pool = (imcs_thread_pool_impl_t*)self; 63 | pool->stop = 1; 64 | imcs_thread_pool_wait(pool); 65 | for (i = 0; i < pool->n_workers; i++) { 66 | pool->workers[i]->join(pool->workers[i]); 67 | } 68 | pool->sync->destroy(pool->sync); 69 | pool->mutex->destroy(pool->sync); 70 | pool->start->destroy(pool->start); 71 | pool->finish->destroy(pool->finish); 72 | free(pool->workers); 73 | free(pool); 74 | } 75 | 76 | static int imcs_thread_pool_get_number_of_threads(struct imcs_thread_pool_t* self) 77 | { 78 | imcs_thread_pool_impl_t* pool = (imcs_thread_pool_impl_t*)self; 79 | return pool->n_workers; 80 | } 81 | 82 | static void imcs_thread_pool_merge(struct imcs_thread_pool_t* self, imcs_job_callback_t callback, void* result) 83 | { 84 | imcs_thread_pool_impl_t* pool = (imcs_thread_pool_impl_t*)self; 85 | pool->sync->lock(pool->sync); 86 | callback(pool->arg, result); 87 | pool->sync->unlock(pool->sync); 88 | } 89 | 90 | struct imcs_thread_pool_t* imcs_create_thread_pool(int n_threads) 91 | { 92 | int i; 93 | imcs_thread_pool_impl_t* pool = (imcs_thread_pool_impl_t*)malloc(sizeof(imcs_thread_pool_impl_t)); 94 | if (n_threads == 0) { 95 | n_threads = imcs_get_number_of_cpus(); 96 | } 97 | pool->workers = (imcs_thread_t**)malloc(sizeof(imcs_thread_t*)*n_threads); 98 | pool->sync = imcs_create_mutex(); 99 | pool->mutex = imcs_create_mutex(); 100 | pool->start = imcs_create_semaphore(0); 101 | pool->finish = imcs_create_semaphore(0); 102 | pool->n_workers = n_threads; 103 | pool->vtab.execute = imcs_thread_pool_execute; 104 | pool->vtab.merge = imcs_thread_pool_merge; 105 | pool->vtab.destroy = imcs_thread_pool_destroy; 106 | pool->vtab.get_number_of_threads = imcs_thread_pool_get_number_of_threads; 107 | 108 | pool->stop = 0; 109 | for (i = 0; i < n_threads; i++) { 110 | pool->workers[i] = imcs_create_thread((imcs_thread_proc_t)imcs_thread_pool_worker, pool); 111 | } 112 | return (struct imcs_thread_pool_t*)pool; 113 | } 114 | -------------------------------------------------------------------------------- /tpch.sql: -------------------------------------------------------------------------------- 1 | create table lineitem( 2 | l_orderkey integer, 3 | l_partkey integer, 4 | l_suppkey integer, 5 | l_linenumber integer, 6 | l_quantity real, 7 | l_extendedprice real, 8 | l_discount real, 9 | l_tax real, 10 | l_returnflag char, 11 | l_linestatus char, 12 | l_shipdate date, 13 | l_commitdate date, 14 | l_receiptdate date, 15 | l_shipinstruct char(25), 16 | l_shipmode char(10), 17 | l_comment char(44), 18 | l_dummy char(1)); 19 | 20 | create index lineitem_order_fk on lineitem(l_orderkey); 21 | 22 | copy lineitem from 'lineitem.tbl' delimiter '|' csv; 23 | 24 | create view lineitems as select 25 | l_orderkey, 26 | l_returnflag, 27 | l_linestatus, 28 | l_quantity, 29 | l_extendedprice, 30 | l_discount, 31 | l_tax, 32 | l_shipdate 33 | from 34 | lineitem; 35 | 36 | \timing 37 | 38 | select 39 | l_returnflag, 40 | l_linestatus, 41 | sum(l_quantity) as sum_qty, 42 | sum(l_extendedprice) as sum_base_price, 43 | sum(l_extendedprice*(1-l_discount)) as sum_disc_price, 44 | sum(l_extendedprice*(1-l_discount)*(1+l_tax)) as sum_charge, 45 | avg(l_quantity) as avg_qty, 46 | avg(l_extendedprice) as avg_price, 47 | avg(l_discount) as avg_disc, 48 | count(*) as count_order 49 | from 50 | lineitem 51 | where 52 | l_shipdate <= cast('1998-12-01' as date) 53 | group by 54 | l_returnflag, 55 | l_linestatus 56 | order by 57 | l_returnflag, 58 | l_linestatus; 59 | 60 | select cs_cut,sum_qty,sum_base_price,sum_disc_price,sum_charge,sum_qty/count_order as avg_qty,sum_base_price/count_order as avg_price,count_order 61 | from 62 | (select cs_cut(group_by,'i1i1'),agg_val as sum_qty from 63 | (select (cs_project_agg(cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_quantity), 64 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag||l_linestatus)))).* 65 | from lineitems_get()) agg) q1 66 | natural join 67 | (select cs_cut(group_by,'i1i1'),agg_val as sum_base_price from 68 | (select (cs_project_agg(cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_extendedprice), 69 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag||l_linestatus)))).* 70 | from lineitems_get()) agg) q2 71 | natural join 72 | (select cs_cut(group_by,'i1i1'),agg_val as sum_disc_price from 73 | (select (cs_project_agg(cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_extendedprice*(-l_discount+1)), 74 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag||l_linestatus)))).* 75 | from lineitems_get()) agg) q3 76 | natural join 77 | (select cs_cut(group_by,'i1i1'),agg_val as sum_charge from 78 | (select (cs_project_agg(cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_extendedprice*(-l_discount+1)*(l_tax+1)), 79 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag||l_linestatus)))).* 80 | from lineitems_get()) agg) q4 81 | natural join 82 | (select cs_cut(group_by,'i1i1'),agg_val as avg_disc from 83 | (select (cs_project_agg(cs_hash_avg(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_discount), 84 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag||l_linestatus)))).* 85 | from lineitems_get()) agg) q5 86 | natural join 87 | (select cs_cut(group_by,'i1i1'),agg_val as count_order from 88 | (select (cs_project_agg(cs_hash_count(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag||l_linestatus)))).* 89 | from lineitems_get()) agg) q6; 90 | 91 | 92 | select cs_cut(group_by,'i1i1'),agg_val as sum_charge from 93 | (select (cs_project_agg(cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_extendedprice*(-l_discount+1)*(l_tax+1)), 94 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag||l_linestatus)))).* 95 | from lineitems_get()) agg; 96 | 97 | 98 | select 99 | l_returnflag, 100 | l_linestatus, 101 | sum(l_extendedprice*(1-l_discount)*(1+l_tax)) as sum_charge 102 | from 103 | lineitem 104 | where 105 | l_shipdate <= cast('1998-12-01' as date) 106 | group by 107 | l_returnflag, 108 | l_linestatus 109 | order by 110 | l_returnflag, 111 | l_linestatus; 112 | 113 | 114 | .* agg; 115 | 116 | 117 | select cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_quantity), 118 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag*256 + l_linestatus)) as sum_qty, 119 | cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_extendedprice), 120 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag*256 + l_linestatus)) as sum_base_price, 121 | cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_extendedprice*(1-l_discount)), 122 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag*256 + l_linestatus)) as sum_disc_price, 123 | cs_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_extendedprice*(1-l_discount)*(1+l_tax)), 124 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag*256 + l_linestatus)) as sum_charge, 125 | cs_hash_avg(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_quantity), 126 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag*256 + l_linestatus)) as avg_qty, 127 | cs_hash_avg(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_price), 128 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag*256 + l_linestatus)) as avg_price, 129 | cs_hash_avg(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_discount), 130 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag*256 + l_linestatus)) as avg_discount, 131 | 132 | 133 | lineitems_hash_sum(cs_filter(l_shipdate <= cast('1998-12-01' as date), l_quantity), 134 | cs_filter(l_shipdate <= cast('1998-12-01' as date), l_returnflag*256 + l_linestatus)) as sum_qty, 135 | 136 | sum(l_quantity) as sum_qty, 137 | sum(l_extendedprice) as sum_base_price, 138 | sum(l_extendedprice*(1-l_discount)) as sum_disc_price, 139 | sum(l_extendedprice*(1-l_discount)*(1+l_tax)) as sum_charge, 140 | avg(l_quantity) as avg_qty, 141 | avg(l_extendedprice) as avg_price, 142 | avg(l_discount) as avg_disc, 143 | count(*) as count_order 144 | f --------------------------------------------------------------------------------