├── .env.example ├── .github ├── ISSUE_TEMPLATE │ ├── 1.general_bug_report.yaml │ └── config.yml ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── benchmark.yaml │ ├── build_spilo.yaml │ └── ci.yaml ├── .gitignore ├── .golangci_lint_version ├── .images ├── FasterthanPG.png ├── ReadmeBenchmarks.png └── header.png ├── BENCHMARKS.md ├── CHANGELOG.md ├── DEVELOPERS.md ├── Dockerfile ├── Dockerfile.spilo ├── HYDRA_PROD_VER ├── LICENSE ├── Makefile ├── README.md ├── VERSION ├── acceptance ├── .golangci.yml ├── fixtures │ └── mysql.sql ├── go.mod ├── go.sum ├── postgres │ └── main_test.go ├── shared │ ├── cases.go │ ├── pg.go │ ├── runner.go │ └── shared.go └── spilo │ └── main_test.go ├── bin └── autolink_changelog.rb ├── columnar ├── .gitattributes ├── .gitignore ├── .ignore ├── Dockerfile ├── LICENSE ├── Makefile ├── Makefile.global.in ├── NOTICE ├── README.md ├── aclocal.m4 ├── autogen.sh ├── cgmanifest.json ├── config │ ├── config.guess │ └── general.m4 ├── configure ├── configure.in ├── docker-build.sh ├── prep_buildtree ├── src │ ├── backend │ │ ├── .gitignore │ │ └── columnar │ │ │ ├── .gitattributes │ │ │ ├── .gitignore │ │ │ ├── Makefile │ │ │ ├── README.md │ │ │ ├── columnar.c │ │ │ ├── columnar.control │ │ │ ├── columnar_cache.c │ │ │ ├── columnar_compression.c │ │ │ ├── columnar_customscan.c │ │ │ ├── columnar_debug.c │ │ │ ├── columnar_indexscan.c │ │ │ ├── columnar_metadata.c │ │ │ ├── columnar_planner_hook.c │ │ │ ├── columnar_read_state_cache.c │ │ │ ├── columnar_reader.c │ │ │ ├── columnar_storage.c │ │ │ ├── columnar_tableam.c │ │ │ ├── columnar_writer.c │ │ │ ├── mod.c │ │ │ ├── safeclib │ │ │ ├── abort_handler_s.c │ │ │ ├── ignore_handler_s.c │ │ │ ├── mem_primitives_lib.c │ │ │ ├── mem_primitives_lib.h │ │ │ ├── memcmp16_s.c │ │ │ ├── memcmp32_s.c │ │ │ ├── memcmp_s.c │ │ │ ├── memcpy16_s.c │ │ │ ├── memcpy32_s.c │ │ │ ├── memcpy_s.c │ │ │ ├── memmove16_s.c │ │ │ ├── memmove32_s.c │ │ │ ├── memmove_s.c │ │ │ ├── memset16_s.c │ │ │ ├── memset32_s.c │ │ │ ├── memset_s.c │ │ │ ├── memzero16_s.c │ │ │ ├── memzero32_s.c │ │ │ ├── memzero_s.c │ │ │ ├── safe_mem_constraint.c │ │ │ ├── safe_mem_constraint.h │ │ │ ├── safe_str_constraint.c │ │ │ ├── safe_str_constraint.h │ │ │ ├── safeclib_private.h │ │ │ ├── snprintf_support.c │ │ │ ├── stpcpy_s.c │ │ │ ├── stpncpy_s.c │ │ │ ├── strcasecmp_s.c │ │ │ ├── strcasestr_s.c │ │ │ ├── strcat_s.c │ │ │ ├── strcmp_s.c │ │ │ ├── strcmpfld_s.c │ │ │ ├── strcpy_s.c │ │ │ ├── strcpyfld_s.c │ │ │ ├── strcpyfldin_s.c │ │ │ ├── strcpyfldout_s.c │ │ │ ├── strcspn_s.c │ │ │ ├── strfirstchar_s.c │ │ │ ├── strfirstdiff_s.c │ │ │ ├── strfirstsame_s.c │ │ │ ├── strisalphanumeric_s.c │ │ │ ├── strisascii_s.c │ │ │ ├── strisdigit_s.c │ │ │ ├── strishex_s.c │ │ │ ├── strislowercase_s.c │ │ │ ├── strismixedcase_s.c │ │ │ ├── strispassword_s.c │ │ │ ├── strisuppercase_s.c │ │ │ ├── strlastchar_s.c │ │ │ ├── strlastdiff_s.c │ │ │ ├── strlastsame_s.c │ │ │ ├── strljustify_s.c │ │ │ ├── strncat_s.c │ │ │ ├── strncpy_s.c │ │ │ ├── strnlen_s.c │ │ │ ├── strnterminate_s.c │ │ │ ├── strpbrk_s.c │ │ │ ├── strprefix_s.c │ │ │ ├── strremovews_s.c │ │ │ ├── strspn_s.c │ │ │ ├── strstr_s.c │ │ │ ├── strtok_s.c │ │ │ ├── strtolowercase_s.c │ │ │ ├── strtouppercase_s.c │ │ │ ├── strzero_s.c │ │ │ ├── wcpcpy_s.c │ │ │ ├── wcscat_s.c │ │ │ ├── wcscpy_s.c │ │ │ ├── wcsncat_s.c │ │ │ ├── wcsncpy_s.c │ │ │ ├── wcsnlen_s.c │ │ │ ├── wmemcmp_s.c │ │ │ ├── wmemcpy_s.c │ │ │ ├── wmemmove_s.c │ │ │ └── wmemset_s.c │ │ │ ├── sql │ │ │ ├── columnar--10.0-1--10.0-2.sql │ │ │ ├── columnar--10.0-2--10.0-3.sql │ │ │ ├── columnar--10.0-3--10.1-1.sql │ │ │ ├── columnar--10.1-1--10.2-1.sql │ │ │ ├── columnar--10.2-1--10.2-2.sql │ │ │ ├── columnar--10.2-2--10.2-3.sql │ │ │ ├── columnar--10.2-3--10.2-4.sql │ │ │ ├── columnar--10.2-4--11.1-1.sql │ │ │ ├── columnar--11.1-1--11.1-2.sql │ │ │ ├── columnar--11.1-10--11.1-11.sql │ │ │ ├── columnar--11.1-11--11.1-12.sql │ │ │ ├── columnar--11.1-2--11.1-3.sql │ │ │ ├── columnar--11.1-3--11.1-4.sql │ │ │ ├── columnar--11.1-4--11.1-5.sql │ │ │ ├── columnar--11.1-5--11.1-6.sql │ │ │ ├── columnar--11.1-6--11.1-7.sql │ │ │ ├── columnar--11.1-7--11.1-8.sql │ │ │ ├── columnar--11.1-8--11.1-9.sql │ │ │ ├── columnar--11.1-9--11.1-10.sql │ │ │ ├── columnar--9.5-1--10.0-1.sql │ │ │ ├── columnar--9.5-1.sql │ │ │ ├── downgrades │ │ │ │ ├── columnar--10.0-1--9.5-1.sql │ │ │ │ ├── columnar--10.0-2--10.0-1.sql │ │ │ │ ├── columnar--10.1-1--10.0-3.sql │ │ │ │ ├── columnar--10.2-1--10.1-1.sql │ │ │ │ ├── columnar--10.2-2--10.2-1.sql │ │ │ │ ├── columnar--10.2-3--10.2-2.sql │ │ │ │ ├── columnar--10.2-4--10.2-3.sql │ │ │ │ ├── columnar--11.1-1--10.2-4.sql │ │ │ │ ├── columnar--11.1-2--11.1-1.sql │ │ │ │ ├── columnar--11.1-3--11.1-2.sql │ │ │ │ ├── columnar--11.1-4--11.1-3.sql │ │ │ │ └── columnar--11.1-5--11.1-4.sql │ │ │ └── udfs │ │ │ │ ├── alter_columnar_table_reset │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 11.1-2.sql │ │ │ │ ├── 11.1-3.sql │ │ │ │ └── latest.sql │ │ │ │ ├── alter_columnar_table_set │ │ │ │ ├── 10.0-1.sql │ │ │ │ ├── 11.1-2.sql │ │ │ │ ├── 11.1-3.sql │ │ │ │ └── latest.sql │ │ │ │ ├── alter_table_set_access_method │ │ │ │ ├── 11.1-11.sql │ │ │ │ ├── 11.1-12.sql │ │ │ │ ├── 11.1-2.sql │ │ │ │ ├── 11.1-4.sql │ │ │ │ ├── 11.1-5.sql │ │ │ │ ├── 11.1-8.sql │ │ │ │ └── latest.sql │ │ │ │ ├── columnar_ensure_am_depends_catalog │ │ │ │ ├── 10.2-4.sql │ │ │ │ └── latest.sql │ │ │ │ ├── columnar_ensure_objects_exist │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ │ ├── columnar_handler │ │ │ │ ├── 10.0-1.sql │ │ │ │ └── latest.sql │ │ │ │ ├── create_table_row_mask │ │ │ │ ├── 11.1-5.sql │ │ │ │ └── latest.sql │ │ │ │ ├── downgrade_columnar_storage │ │ │ │ ├── 10.2-1.sql │ │ │ │ └── latest.sql │ │ │ │ ├── upgrade_columnar_storage │ │ │ │ ├── 10.2-1.sql │ │ │ │ └── latest.sql │ │ │ │ └── vacuum │ │ │ │ ├── 11.1-10.sql │ │ │ │ ├── 11.1-6.sql │ │ │ │ ├── 11.1-7.sql │ │ │ │ └── latest.sql │ │ │ ├── vectorization │ │ │ ├── columnar_vector_execution.c │ │ │ ├── columnar_vector_types.c │ │ │ ├── nodes │ │ │ │ └── columnar_aggregator_node.c │ │ │ └── types │ │ │ │ ├── aggregates.c │ │ │ │ ├── date.c │ │ │ │ ├── int.c │ │ │ │ └── numeric.c │ │ │ ├── write_state_interface.c │ │ │ ├── write_state_management.c │ │ │ └── write_state_row_mask.c │ ├── include │ │ ├── .gitignore │ │ ├── citus_config.h.in │ │ ├── citus_version.h.in │ │ ├── columnar │ │ │ ├── columnar.h │ │ │ ├── columnar_compression.h │ │ │ ├── columnar_customscan.h │ │ │ ├── columnar_indexscan.h │ │ │ ├── columnar_metadata.h │ │ │ ├── columnar_storage.h │ │ │ ├── columnar_tableam.h │ │ │ ├── columnar_version_compat.h │ │ │ ├── columnar_write_state_row_mask.h │ │ │ ├── utils │ │ │ │ └── listutils.h │ │ │ └── vectorization │ │ │ │ ├── columnar_vector_execution.h │ │ │ │ ├── columnar_vector_types.h │ │ │ │ ├── nodes │ │ │ │ └── columnar_aggregator_node.h │ │ │ │ └── types │ │ │ │ ├── numeric.h │ │ │ │ └── types.h │ │ ├── pg_version_compat.h │ │ └── pg_version_constants.h │ └── test │ │ └── regress │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── README.md │ │ ├── columnar_regression.conf │ │ ├── columnar_schedule │ │ ├── data │ │ ├── array_types.csv │ │ ├── contestants.1.csv │ │ ├── contestants.2.csv │ │ ├── datetime_types.csv │ │ ├── enum_and_composite_types.csv │ │ ├── null_values.csv │ │ ├── other_types.csv │ │ └── range_types.csv │ │ ├── expected │ │ ├── .gitignore │ │ ├── columnar_aggregates.out │ │ ├── columnar_aggregates_1.out │ │ ├── columnar_aggregates_2.out │ │ ├── columnar_alter.out │ │ ├── columnar_alter_set_type.out │ │ ├── columnar_alter_table_set_access_method.out │ │ ├── columnar_analyze.out │ │ ├── columnar_cache.out │ │ ├── columnar_cache_1.out │ │ ├── columnar_chunk_filtering.out │ │ ├── columnar_clean.out │ │ ├── columnar_create.out │ │ ├── columnar_cursor.out │ │ ├── columnar_customindex.out │ │ ├── columnar_customindex_1.out │ │ ├── columnar_drop.out │ │ ├── columnar_empty.out │ │ ├── columnar_fallback_scan.out │ │ ├── columnar_first_row_number.out │ │ ├── columnar_indexes.out │ │ ├── columnar_insert.out │ │ ├── columnar_insert_1.out │ │ ├── columnar_insert_2.out │ │ ├── columnar_join.out │ │ ├── columnar_lz4.out │ │ ├── columnar_matview.out │ │ ├── columnar_memory.out │ │ ├── columnar_partitioning.out │ │ ├── columnar_paths.out │ │ ├── columnar_paths_1.out │ │ ├── columnar_paths_2.out │ │ ├── columnar_permissions.out │ │ ├── columnar_query.out │ │ ├── columnar_recursive.out │ │ ├── columnar_rollback.out │ │ ├── columnar_tableoptions.out │ │ ├── columnar_test_helpers.out │ │ ├── columnar_transactions.out │ │ ├── columnar_trigger.out │ │ ├── columnar_truncate.out │ │ ├── columnar_types_without_comparison.out │ │ ├── columnar_update_delete.out │ │ ├── columnar_upsert.out │ │ ├── columnar_vacuum.out │ │ ├── columnar_vacuum_udf.out │ │ ├── columnar_vacuum_udf_1.out │ │ ├── columnar_vectorization.out │ │ └── columnar_zstd.out │ │ ├── input │ │ ├── columnar_copyto.source │ │ ├── columnar_data_types.source │ │ └── columnar_load.source │ │ ├── output │ │ ├── columnar_copyto.source │ │ ├── columnar_data_types.source │ │ └── columnar_load.source │ │ └── sql │ │ ├── .gitignore │ │ ├── columnar_aggregates.sql │ │ ├── columnar_alter.sql │ │ ├── columnar_alter_set_type.sql │ │ ├── columnar_alter_table_set_access_method.sql │ │ ├── columnar_analyze.sql │ │ ├── columnar_cache.sql │ │ ├── columnar_chunk_filtering.sql │ │ ├── columnar_clean.sql │ │ ├── columnar_create.sql │ │ ├── columnar_cursor.sql │ │ ├── columnar_customindex.sql │ │ ├── columnar_drop.sql │ │ ├── columnar_empty.sql │ │ ├── columnar_fallback_scan.sql │ │ ├── columnar_first_row_number.sql │ │ ├── columnar_indexes.sql │ │ ├── columnar_insert.sql │ │ ├── columnar_join.sql │ │ ├── columnar_lz4.sql │ │ ├── columnar_matview.sql │ │ ├── columnar_memory.sql │ │ ├── columnar_partitioning.sql │ │ ├── columnar_paths.sql │ │ ├── columnar_permissions.sql │ │ ├── columnar_query.sql │ │ ├── columnar_recursive.sql │ │ ├── columnar_rollback.sql │ │ ├── columnar_tableoptions.sql │ │ ├── columnar_test_helpers.sql │ │ ├── columnar_transactions.sql │ │ ├── columnar_trigger.sql │ │ ├── columnar_truncate.sql │ │ ├── columnar_types_without_comparison.sql │ │ ├── columnar_update_delete.sql │ │ ├── columnar_upsert.sql │ │ ├── columnar_vacuum.sql │ │ ├── columnar_vacuum_udf.sql │ │ ├── columnar_vectorization.sql │ │ └── columnar_zstd.sql └── vendor │ ├── README.md │ └── safestringlib │ ├── .gitattributes │ ├── .gitignore │ ├── CMakeLists.txt │ ├── CODE_OF_CONDUCT.md │ ├── LICENSE │ ├── LICENSE©ING.txt │ ├── README.md │ ├── include │ ├── safe_lib.h │ ├── safe_lib_errno.h │ ├── safe_lib_errno.h.in │ ├── safe_mem_lib.h │ ├── safe_str_lib.h │ ├── safe_types.h │ ├── safe_types.h.in │ └── snprintf_s.h │ ├── makefile │ └── safeclib │ ├── abort_handler_s.c │ ├── ignore_handler_s.c │ ├── mem_primitives_lib.c │ ├── mem_primitives_lib.h │ ├── memcmp16_s.c │ ├── memcmp32_s.c │ ├── memcmp_s.c │ ├── memcpy16_s.c │ ├── memcpy32_s.c │ ├── memcpy_s.c │ ├── memmove16_s.c │ ├── memmove32_s.c │ ├── memmove_s.c │ ├── memset16_s.c │ ├── memset32_s.c │ ├── memset_s.c │ ├── memzero16_s.c │ ├── memzero32_s.c │ ├── memzero_s.c │ ├── safe_mem_constraint.c │ ├── safe_mem_constraint.h │ ├── safe_str_constraint.c │ ├── safe_str_constraint.h │ ├── safeclib_private.h │ ├── snprintf_support.c │ ├── stpcpy_s.c │ ├── stpncpy_s.c │ ├── strcasecmp_s.c │ ├── strcasestr_s.c │ ├── strcat_s.c │ ├── strcmp_s.c │ ├── strcmpfld_s.c │ ├── strcpy_s.c │ ├── strcpyfld_s.c │ ├── strcpyfldin_s.c │ ├── strcpyfldout_s.c │ ├── strcspn_s.c │ ├── strfirstchar_s.c │ ├── strfirstdiff_s.c │ ├── strfirstsame_s.c │ ├── strisalphanumeric_s.c │ ├── strisascii_s.c │ ├── strisdigit_s.c │ ├── strishex_s.c │ ├── strislowercase_s.c │ ├── strismixedcase_s.c │ ├── strispassword_s.c │ ├── strisuppercase_s.c │ ├── strlastchar_s.c │ ├── strlastdiff_s.c │ ├── strlastsame_s.c │ ├── strljustify_s.c │ ├── strncat_s.c │ ├── strncpy_s.c │ ├── strnlen_s.c │ ├── strnterminate_s.c │ ├── strpbrk_s.c │ ├── strprefix_s.c │ ├── strremovews_s.c │ ├── strspn_s.c │ ├── strstr_s.c │ ├── strtok_s.c │ ├── strtolowercase_s.c │ ├── strtouppercase_s.c │ ├── strzero_s.c │ ├── wcpcpy_s.c │ ├── wcscat_s.c │ ├── wcscpy_s.c │ ├── wcsncat_s.c │ ├── wcsncpy_s.c │ ├── wcsnlen_s.c │ ├── wmemcmp_s.c │ ├── wmemcpy_s.c │ ├── wmemmove_s.c │ └── wmemset_s.c ├── docker-bake.hcl ├── docker-compose.yml ├── files ├── postgres │ ├── docker-entrypoint-initdb.d │ │ └── columnar.sh │ └── postgresql.conf └── spilo │ └── postgres-appliance │ └── scripts │ ├── configure_spilo.py │ ├── multicorn │ └── after-create.sql │ ├── mysql_fdw │ └── after-create.sql │ ├── parquet_s3_fdw │ └── after-create.sql │ ├── post_init.sh │ └── spilo_commons.py └── third-party └── pgxman ├── pgxman_13.yaml ├── pgxman_13_spilo.yaml ├── pgxman_14.yaml ├── pgxman_14_spilo.yaml ├── pgxman_15.yaml └── pgxman_16.yaml /.env.example: -------------------------------------------------------------------------------- 1 | POSTGRES_PORT=5432 2 | POSTGRES_USER=postgres 3 | POSTGRES_PASSWORD=hydra 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/1.general_bug_report.yaml: -------------------------------------------------------------------------------- 1 | name: Bug Report 2 | description: File a bug report 3 | title: "[Bug]: " 4 | labels: ["bug", "triage"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | Thanks for taking the time to fill out this bug report! Please ensure you've read our [documentation](https://docs.hydra.so/). If you're unsure as to whether or not you're experiencing a bug, please reach out on our [Discord](https://hydra.so/discord) to gain a better understanding of what you're facing. 10 | - type: textarea 11 | attributes: 12 | label: "What's wrong?" 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: 📖 Read the documentation 4 | url: https://docs.hydra.so/ 5 | about: Please consult the documentation before opening any issues! 6 | - name: 🙋 Ask a question on Discord 7 | url: https://hydra.so/discord 8 | about: Ask questions about how to use Hydra on our Discord. 9 | - name: 📝 Open a documentation issue or PR 10 | url: https://github.com/hydradatabase/docs 11 | about: You can open issues or PRs directly against our documentation. 12 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 14 | 15 | 16 | 17 | ### What's changed? 18 | 19 | 25 | -------------------------------------------------------------------------------- /.github/workflows/benchmark.yaml: -------------------------------------------------------------------------------- 1 | name: Benchmark 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | repo: 7 | required: true 8 | type: string 9 | default: ghcr.io/hydradatabase/hydra 10 | benchmark: 11 | required: true 12 | type: string 13 | tag: 14 | required: true 15 | type: string 16 | 17 | workflow_dispatch: 18 | inputs: 19 | repo: 20 | required: true 21 | type: choice 22 | default: ghcr.io/hydradatabase/hydra 23 | options: 24 | - ghcr.io/hydradatabase/hydra 25 | - 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo 26 | benchmark: 27 | required: true 28 | type: choice 29 | default: clickbench-1M 30 | options: 31 | - clickbench-10k 32 | - clickbench-1M 33 | - clickbench-100M 34 | - warehouse-1G 35 | - warehouse-10G 36 | tag: 37 | type: string 38 | env: 39 | BENCHMARK: ${{ inputs.benchmark || 'clickbench-1M' }} 40 | REPO: ${{ inputs.repo || 'ghcr.io/hydradatabase/hydra' }} 41 | TAG: ${{ inputs.tag || format('15-{0}', github.sha) }} 42 | BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} 43 | BENCHER_TESTBED: gh-4core 44 | 45 | jobs: 46 | benchmarks: 47 | 48 | name: Run Benchmark 49 | # 16gb ram, 4vcpu, 150gb disk 50 | runs-on: benchmarks-ubuntu-latest-4core 51 | 52 | steps: 53 | - uses: actions/setup-node@v3 54 | with: 55 | node-version: 18 56 | 57 | - uses: docker/setup-buildx-action@v2 58 | 59 | - uses: unfor19/install-aws-cli-action@v1 60 | with: 61 | arch: amd64 62 | 63 | - uses: bencherdev/bencher@main 64 | 65 | - name: Checkout benchmarks 66 | uses: actions/checkout@v3 67 | with: 68 | repository: hydradatabase/benchmarks 69 | 70 | - name: Configure AWS credentials 71 | uses: aws-actions/configure-aws-credentials@v1 72 | with: 73 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY || secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }} 74 | aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY || secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }} 75 | aws-region: us-east-1 76 | mask-aws-account-id: no 77 | 78 | - name: Login to Amazon ECR 79 | if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }} 80 | id: spilo-ecr 81 | uses: aws-actions/amazon-ecr-login@v1 82 | with: 83 | registries: "011789831835" 84 | 85 | - name: Run benchmark 86 | run: ./run-gha.sh 87 | -------------------------------------------------------------------------------- /.github/workflows/build_spilo.yaml: -------------------------------------------------------------------------------- 1 | name: Build Spilo 2 | 3 | on: 4 | workflow_call: 5 | inputs: 6 | production: 7 | required: true 8 | type: boolean 9 | default: false 10 | spilo_repo: 11 | required: true 12 | type: string 13 | workflow_dispatch: 14 | inputs: 15 | production: 16 | required: true 17 | type: boolean 18 | default: false 19 | spilo_repo: 20 | required: true 21 | type: string 22 | default: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo 23 | 24 | jobs: 25 | push_spilo: 26 | if: github.repository == 'hydradatabase/hydra-internal' 27 | name: Push Spilo 28 | runs-on: ubuntu-latest 29 | timeout-minutes: 15 30 | steps: 31 | - name: Checkout 32 | uses: actions/checkout@v3 33 | - name: Set up Depot 34 | uses: depot/setup-action@v1 35 | - name: Configure AWS credentials 36 | uses: aws-actions/configure-aws-credentials@v1 37 | with: 38 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} 39 | aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }} 40 | aws-region: us-east-1 41 | mask-aws-account-id: no 42 | - name: Login to Amazon ECR 43 | id: spilo-ecr 44 | uses: aws-actions/amazon-ecr-login@v1 45 | with: 46 | registries: "011789831835" 47 | - name: Bake and push spilo 48 | uses: depot/bake-action@v1 49 | with: 50 | token: ${{ secrets.DEPOT_TOKEN }} 51 | project: ${{ secrets.DEPOT_PROJECT }} 52 | push: true 53 | targets: spilo 54 | set: | 55 | spilo.tags=${{ inputs.spilo_repo }}:${{ inputs.production && github.sha || format('dev-{0}', github.sha) }} 56 | ${{ inputs.production && format('spilo.tags={0}:latest', inputs.spilo_repo) || '' }} 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.env 3 | /volumes 4 | /tmp 5 | -------------------------------------------------------------------------------- /.golangci_lint_version: -------------------------------------------------------------------------------- 1 | v1.54.2 2 | -------------------------------------------------------------------------------- /.images/FasterthanPG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydradatabase/columnar/d1f7657ee82265b2b9f1c06fda929bb5b942411b/.images/FasterthanPG.png -------------------------------------------------------------------------------- /.images/ReadmeBenchmarks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydradatabase/columnar/d1f7657ee82265b2b9f1c06fda929bb5b942411b/.images/ReadmeBenchmarks.png -------------------------------------------------------------------------------- /.images/header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydradatabase/columnar/d1f7657ee82265b2b9f1c06fda929bb5b942411b/.images/header.png -------------------------------------------------------------------------------- /DEVELOPERS.md: -------------------------------------------------------------------------------- 1 | # Developing Hydra 2 | 3 | Most of the work on Hydra exists outside of this repo through various extensions that we add to the image. 4 | The hydra project exists for coordination and to build the final Docker image which contains Postgres and 5 | these extensions. 6 | 7 | Currently active projects are: 8 | 9 | * [the Hydra columnar engine](https://github.com/hydradatabase/citus) 10 | 11 | ## Build 12 | 13 | The Hydra Docker image is based on [zalando/spilo](https://github.com/zalando/spilo). 14 | The image is a distribution of Spilo and the [Columnar PostgreSQL extension](https://github.com/hydradatabase/citus). 15 | 16 | To build, run: 17 | 18 | ``` 19 | TAG=1234 TARGET=hydra make docker_build 20 | ``` 21 | 22 | ## Image Build Tags 23 | 24 | Image build tag is in the format of `${SPILO_SHA}_${COLUMNAR_EXT_SHA}`, e.g. `72fb97e_ff32dd9`. 25 | The `latest` tag is always tagged to the latest main branch. 26 | 27 | ## Hydra Production Version 28 | 29 | The `HYDRA_PROD_VER` file stores the version that Hydra runs in production. 30 | Please bump it when a new production version is released. 31 | 32 | ## Spilo Version Update 33 | 34 | Hydra Docker build overrides the following Spilo scripts to enable extra PostgreSQL extensions: 35 | 36 | * [configure_spilo.py](https://github.com/zalando/spilo/blob/master/postgres-appliance/scripts/configure_spilo.py) 37 | * [spilo_commons.py](https://github.com/zalando/spilo/blob/master/postgres-appliance/scripts/spilo_commons.py) 38 | * [post_init.sh](https://github.com/zalando/spilo/blob/master/postgres-appliance/scripts/post_init.sh) 39 | 40 | To update to a newer Spilo version, you need to copy the above files from the specific Spilo version and add the corresponding bits to enable the Columnar extensions. 41 | Make sure you use `diff` to understand what needs to be added first before updating. 42 | In the future, we may provide a script to simplify the update process. 43 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | #syntax=docker/dockerfile:1 2 | 3 | FROM postgres_base 4 | 5 | RUN set -eux; \ 6 | apt-get update; \ 7 | apt-get install -y --no-install-recommends \ 8 | curl \ 9 | ca-certificates \ 10 | ; \ 11 | rm -rf /var/lib/apt/lists/* 12 | 13 | # columnar ext 14 | # NOTE(owenthereal): build columnar with pgxman in this repo 15 | COPY --from=columnar /pg_ext / 16 | 17 | COPY files/postgres/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d/ 18 | 19 | ARG POSTGRES_BASE_VERSION 20 | # Install pgxman extensions 21 | # Always force rebuild of this layer 22 | ARG TIMESTAMP=1 23 | COPY third-party/pgxman /tmp/pgxman/ 24 | RUN curl -sfL https://install.pgx.sh | sh -s -- /tmp/pgxman/pgxman_${POSTGRES_BASE_VERSION}.yaml && \ 25 | pgxman install pgsql-http=1.6.0 --pg ${POSTGRES_BASE_VERSION} --yes && \ 26 | rm -rf /tmp/pgxman 27 | -------------------------------------------------------------------------------- /Dockerfile.spilo: -------------------------------------------------------------------------------- 1 | #syntax=docker/dockerfile:1 2 | 3 | FROM spilo_base 4 | 5 | RUN set -eux; \ 6 | apt-get update; \ 7 | apt-get install -y --no-install-recommends \ 8 | curl \ 9 | ca-certificates \ 10 | ; \ 11 | rm -rf /var/lib/apt/lists/* 12 | 13 | # See https://github.com/docker-library/postgres/issues/415 14 | RUN set -eux; \ 15 | locale-gen en_US.UTF-8; \ 16 | echo "LC_ALL=en_US.UTF-8" >> /etc/environment; \ 17 | echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen; \ 18 | echo "LANG=en_US.UTF-8" > /etc/locale.conf 19 | 20 | # columnar ext 21 | # NOTE(owenthereal): build columnar with pgxman in this repo 22 | COPY --from=columnar_13 /pg_ext / 23 | COPY --from=columnar_14 /pg_ext / 24 | COPY --from=columnar_15 /pg_ext / 25 | 26 | # configuration 27 | COPY files/spilo/postgres-appliance/scripts /scripts/ 28 | 29 | ARG POSTGRES_BASE_VERSION 30 | # Default envs 31 | ENV PGVERSION=${POSTGRES_BASE_VERSION} SPILO_PROVIDER=local PGUSER_SUPERUSER=postgres PGPASSWORD_SUPERUSER=hydra 32 | 33 | # Install pgxman extensions 34 | # Always force rebuild of this layer 35 | ARG TIMESTAMP=1 36 | COPY third-party/pgxman /tmp/pgxman/ 37 | RUN curl -sfL https://install.pgx.sh | sh -s -- /tmp/pgxman/pgxman_13_spilo.yaml /tmp/pgxman/pgxman_14_spilo.yaml && \ 38 | rm -rf /tmp/pgxman 39 | -------------------------------------------------------------------------------- /HYDRA_PROD_VER: -------------------------------------------------------------------------------- 1 | 3cdf8367ae8717e67c60daf4e5eafd7134de589a 2 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.1.2 2 | -------------------------------------------------------------------------------- /acceptance/.golangci.yml: -------------------------------------------------------------------------------- 1 | linters: 2 | presets: 3 | - bugs 4 | - comment 5 | - error 6 | - import 7 | - sql 8 | - test 9 | - unused 10 | disable: 11 | # unwanted 12 | - depguard 13 | - exhaustivestruct 14 | - exhaustruct 15 | - gosec 16 | - paralleltest 17 | # deprecated 18 | - deadcode 19 | - scopelint 20 | - varcheck 21 | # issues with generics 22 | - rowserrcheck 23 | - sqlclosecheck 24 | - structcheck 25 | -------------------------------------------------------------------------------- /acceptance/fixtures/mysql.sql: -------------------------------------------------------------------------------- 1 | create database test; 2 | use test; 3 | 4 | CREATE TABLE warehouse ( 5 | warehouse_id int primary key, 6 | warehouse_name varchar(255), 7 | warehouse_created timestamp 8 | ); 9 | -------------------------------------------------------------------------------- /acceptance/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hydradatabase/hydra/acceptance 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/google/uuid v1.3.0 7 | github.com/jackc/pgx/v5 v5.0.4 8 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd 9 | github.com/rs/xid v1.2.1 10 | ) 11 | 12 | require ( 13 | github.com/jackc/pgpassfile v1.0.0 // indirect 14 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect 15 | github.com/jackc/puddle/v2 v2.0.0 // indirect 16 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect 17 | golang.org/x/text v0.3.8 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /acceptance/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 5 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 6 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= 7 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= 8 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= 9 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= 10 | github.com/jackc/pgx/v5 v5.0.4 h1:r5O6y84qHX/z/HZV40JBdx2obsHz7/uRj5b+CcYEdeY= 11 | github.com/jackc/pgx/v5 v5.0.4/go.mod h1:U0ynklHtgg43fue9Ly30w3OCSTDPlXjig9ghrNGaguQ= 12 | github.com/jackc/puddle/v2 v2.0.0 h1:Kwk/AlLigcnZsDssc3Zun1dk1tAtQNPaBBxBHWn0Mjc= 13 | github.com/jackc/puddle/v2 v2.0.0/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc= 14 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd h1:nIzoSW6OhhppWLm4yqBwZsKJlAayUu5FGozhrF3ETSM= 15 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd/go.mod h1:MEQrHur0g8VplbLOv5vXmDzacSaH9Z7XhcgsSh1xciU= 16 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 18 | github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= 19 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 20 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 21 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 22 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 23 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= 24 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 25 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= 26 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 27 | golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= 28 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 29 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 30 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 31 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 32 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 33 | -------------------------------------------------------------------------------- /acceptance/shared/pg.go: -------------------------------------------------------------------------------- 1 | package shared 2 | 3 | import ( 4 | "context" 5 | "regexp" 6 | "testing" 7 | 8 | "github.com/jackc/pgx/v5/pgxpool" 9 | ) 10 | 11 | var ( 12 | regexpPGVersion = regexp.MustCompile(`^PostgreSQL (\d+)`) 13 | ) 14 | 15 | func QueryPGVersion(t *testing.T, ctx context.Context, pool *pgxpool.Pool) PGVersion { 16 | row := pool.QueryRow(ctx, "SELECT VERSION();") 17 | 18 | var version string 19 | if err := row.Scan(&version); err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | matches := regexpPGVersion.FindStringSubmatch(version) 24 | if len(matches) == 0 { 25 | t.Fatalf("failed to parse pg_config --version output: %s", version) 26 | } 27 | 28 | return PGVersion(matches[1]) 29 | } 30 | -------------------------------------------------------------------------------- /bin/autolink_changelog.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # usage: bin/autolink_changelog.rb < CHANGELOG.rb > LINKED-CHANGELOG.rb 4 | 5 | require 'set' 6 | 7 | PR_REGEXP = /(?&2 15 | exit 1 16 | elif test x"$1" = x"--help"; then 17 | echo "$help" 18 | exit 0 19 | fi 20 | 21 | unset CDPATH 22 | 23 | sourcetree=`cd $1 && pwd` 24 | 25 | buildtree=`cd ${2:-'.'} && pwd` 26 | 27 | # We must not auto-create the subdirectories holding built documentation. 28 | # If we did, it would interfere with installation of prebuilt docs from 29 | # the source tree, if a VPATH build is done from a distribution tarball. 30 | # See bug #5595. 31 | for item in `find "$sourcetree" -type d \( \( -name CVS -prune \) -o \( -name .git -prune \) -o -print \) | grep -v "$sourcetree/doc/src/sgml/\+"`; do 32 | subdir=`expr "$item" : "$sourcetree\(.*\)"` 33 | if test ! -d "$buildtree/$subdir"; then 34 | mkdir -p "$buildtree/$subdir" || exit 1 35 | fi 36 | done 37 | 38 | for item in `find "$sourcetree" -not -path '*/.git/hg/*' \( -name Makefile -print -o -name GNUmakefile -print \)`; do 39 | filename=`expr "$item" : "$sourcetree\(.*\)"` 40 | if test ! -f "${item}.in"; then 41 | if cmp "$item" "$buildtree/$filename" >/dev/null 2>&1; then : ; else 42 | ln -fs "$item" "$buildtree/$filename" || exit 1 43 | fi 44 | fi 45 | done 46 | 47 | exit 0 48 | -------------------------------------------------------------------------------- /columnar/src/backend/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hydradatabase/columnar/d1f7657ee82265b2b9f1c06fda929bb5b942411b/columnar/src/backend/.gitignore -------------------------------------------------------------------------------- /columnar/src/backend/columnar/.gitattributes: -------------------------------------------------------------------------------- 1 | * whitespace=space-before-tab,trailing-space 2 | *.[chly] whitespace=space-before-tab,trailing-space,indent-with-non-tab,tabwidth=4 3 | *.dsl whitespace=space-before-tab,trailing-space,tab-in-indent 4 | *.patch -whitespace 5 | *.pl whitespace=space-before-tab,trailing-space,tabwidth=4 6 | *.po whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eof 7 | *.sgml whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eol 8 | *.x[ms]l whitespace=space-before-tab,trailing-space,tab-in-indent 9 | 10 | # Avoid confusing ASCII underlines with leftover merge conflict markers 11 | README conflict-marker-size=32 12 | README.* conflict-marker-size=32 13 | 14 | # Certain data files that contain special whitespace, and other special cases 15 | *.data -whitespace 16 | 17 | # Test output files that contain extra whitespace 18 | *.out -whitespace 19 | src/test/regress/output/*.source -whitespace 20 | 21 | # These files are maintained or generated elsewhere. We take them as is. 22 | configure -whitespace 23 | 24 | # all C files (implementation and header) use our style... 25 | *.[ch] citus-style 26 | 27 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/.gitignore: -------------------------------------------------------------------------------- 1 | /build/ -------------------------------------------------------------------------------- /columnar/src/backend/columnar/Makefile: -------------------------------------------------------------------------------- 1 | citus_subdir = src/backend/columnar 2 | citus_top_builddir = ../../.. 3 | safestringlib_srcdir = $(citus_abs_top_srcdir)/vendor/safestringlib 4 | SUBDIRS = . safeclib vectorization vectorization/types vectorization/nodes 5 | SUBDIRS += 6 | ENSURE_SUBDIRS_EXIST := $(shell mkdir -p $(SUBDIRS)) 7 | OBJS += \ 8 | $(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(sort $(wildcard $(citus_abs_srcdir)/$(dir)/*.c)))) 9 | 10 | MODULE_big = columnar 11 | PG_CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include 12 | 13 | ifdef COLUMNAR_O3 14 | PG_CFLAGS += -O3 15 | endif 16 | 17 | EXTENSION = columnar 18 | 19 | template_sql_files = $(patsubst $(citus_abs_srcdir)/%,%,$(wildcard $(citus_abs_srcdir)/sql/*.sql)) 20 | template_downgrade_sql_files = $(patsubst $(citus_abs_srcdir)/sql/downgrades/%,%,$(wildcard $(citus_abs_srcdir)/sql/downgrades/*.sql)) 21 | generated_sql_files = $(patsubst %,$(citus_abs_srcdir)/build/%,$(template_sql_files)) 22 | generated_downgrade_sql_files += $(patsubst %,$(citus_abs_srcdir)/build/sql/%,$(template_downgrade_sql_files)) 23 | # All citus--*.sql files that are used to upgrade between versions 24 | DATA_built = $(generated_sql_files) 25 | 26 | include $(citus_top_builddir)/Makefile.global 27 | 28 | SQL_DEPDIR=.deps/sql 29 | SQL_BUILDDIR=build/sql 30 | 31 | $(generated_sql_files): $(citus_abs_srcdir)/build/%: % 32 | @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR) 33 | @# -MF is used to store dependency files(.Po) in another directory for separation 34 | @# -MT is used to change the target of the rule emitted by dependency generation. 35 | @# -P is used to inhibit generation of linemarkers in the output from the preprocessor. 36 | @# -undef is used to not predefine any system-specific or GCC-specific macros. 37 | @# `man cpp` for further information 38 | cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ $< > $@ 39 | 40 | $(generated_downgrade_sql_files): $(citus_abs_srcdir)/build/sql/%: sql/downgrades/% 41 | @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR) 42 | @# -MF is used to store dependency files(.Po) in another directory for separation 43 | @# -MT is used to change the target of the rule emitted by dependency generation. 44 | @# -P is used to inhibit generation of linemarkers in the output from the preprocessor. 45 | @# -undef is used to not predefine any system-specific or GCC-specific macros. 46 | @# `man cpp` for further information 47 | cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ $< > $@ 48 | 49 | SQL_Po_files := $(wildcard $(SQL_DEPDIR)/*.Po) 50 | ifneq (,$(SQL_Po_files)) 51 | include $(SQL_Po_files) 52 | endif 53 | 54 | .PHONY: install-all 55 | install-all: install 56 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/columnar.control: -------------------------------------------------------------------------------- 1 | comment = 'Hydra Columnar extension' 2 | default_version = '11.1-12' 3 | module_pathname = '$libdir/columnar' 4 | relocatable = false 5 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/mod.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * mod.c 4 | * 5 | * This file contains module-level definitions. 6 | * 7 | * Copyright (c) 2016, Citus Data, Inc. 8 | * 9 | * $Id$ 10 | * 11 | *------------------------------------------------------------------------- 12 | */ 13 | 14 | #include "postgres.h" 15 | 16 | #include "fmgr.h" 17 | 18 | #include "citus_version.h" 19 | 20 | #include "columnar/columnar.h" 21 | #include "columnar/columnar_tableam.h" 22 | 23 | 24 | PG_MODULE_MAGIC; 25 | 26 | void _PG_init(void); 27 | 28 | void 29 | _PG_init(void) 30 | { 31 | columnar_init(); 32 | } 33 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/safeclib/abort_handler_s.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * abort_handler_s.c 3 | * 4 | * 2012, Jonathan Toppins 5 | * 6 | * Copyright (c) 2012 Cisco Systems 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #include "safeclib_private.h" 33 | 34 | /** 35 | * NAME 36 | * abort_handler_s 37 | * 38 | * SYNOPSIS 39 | * #include "safe_lib.h" 40 | * void abort_handler_s(const char *msg, void *ptr, errno_t error) 41 | * 42 | * DESCRIPTION 43 | * This function writes a message on the standard error stream in 44 | * an implementation-defined format. The message shall include the 45 | * string pointed to by msg. The abort_handler_s function then calls 46 | * the abort function. 47 | * 48 | * SPECIFIED IN 49 | * ISO/IEC JTC1 SC22 WG14 N1172, Programming languages, environments 50 | * and system software interfaces, Extensions to the C Library, 51 | * Part I: Bounds-checking interfaces 52 | * 53 | * INPUT PARAMETERS 54 | * msg Pointer to the message describing the error 55 | * 56 | * ptr Pointer to aassociated data. Can be NULL. 57 | * 58 | * error The error code encountered. 59 | * 60 | * RETURN VALUE 61 | * Does not return to caller. 62 | * 63 | * ALSO SEE 64 | * ignore_handler_s() 65 | * 66 | */ 67 | 68 | void abort_handler_s(const char *msg, void *ptr, errno_t error) 69 | { 70 | slprintf("ABORT CONSTRAINT HANDLER: (%u) %s\n", error, 71 | (msg) ? msg : "Null message"); 72 | slabort(); 73 | } 74 | EXPORT_SYMBOL(abort_handler_s) 75 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/safeclib/ignore_handler_s.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * ignore_handler_s.c 3 | * 4 | * 2012, Jonathan Toppins 5 | * 6 | * Copyright (c) 2012 Cisco Systems 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #include "safeclib_private.h" 33 | 34 | /** 35 | * NAME 36 | * ignore_handler_s 37 | * 38 | * SYNOPSIS 39 | * #include "safe_lib.h" 40 | * void ignore_handler_s(const char *msg, void *ptr, errno_t error) 41 | * 42 | * DESCRIPTION 43 | * This function simply returns to the caller. 44 | * 45 | * SPECIFIED IN 46 | * ISO/IEC JTC1 SC22 WG14 N1172, Programming languages, environments 47 | * and system software interfaces, Extensions to the C Library, 48 | * Part I: Bounds-checking interfaces 49 | * 50 | * INPUT PARAMETERS 51 | * msg Pointer to the message describing the error 52 | * 53 | * ptr Pointer to aassociated data. Can be NULL. 54 | * 55 | * error The error code encountered. 56 | * 57 | * RETURN VALUE 58 | * Returns no value. 59 | * 60 | * ALSO SEE 61 | * abort_handler_s() 62 | * 63 | */ 64 | 65 | void ignore_handler_s(const char *msg, void *ptr, errno_t error) 66 | { 67 | 68 | sldebug_printf("IGNORE CONSTRAINT HANDLER: (%u) %s\n", error, 69 | (msg) ? msg : "Null message"); 70 | return; 71 | } 72 | EXPORT_SYMBOL(ignore_handler_s) 73 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/safeclib/mem_primitives_lib.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * mem_primitives_lib.h - Unguarded Memory Copy Routines 3 | * 4 | * October 2008, Bo Berry 5 | * 6 | * Copyright (c) 2008-2011 by Cisco Systems, Inc 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #ifndef __MEM_PRIMITIVES_LIB_H__ 33 | #define __MEM_PRIMITIVES_LIB_H__ 34 | 35 | #include "safeclib_private.h" 36 | 37 | /* 38 | * These are prototypes for _unguarded_ memory routines. The caller must 39 | * validate all parameters prior to invocation. Useful for diagnostics 40 | * and system initialization processing. 41 | */ 42 | 43 | /* moves (handles overlap) memory */ 44 | extern void 45 | mem_prim_move(void *dest, const void *src, uint32_t length); 46 | 47 | 48 | /* uint8_t moves (handles overlap) memory */ 49 | extern void 50 | mem_prim_move8(uint8_t *dest, const uint8_t *src, uint32_t length); 51 | 52 | /* uint16_t moves (handles overlap) memory */ 53 | extern void 54 | mem_prim_move16(uint16_t *dest, const uint16_t *src, uint32_t length); 55 | 56 | /* uint32_t moves (handles overlap) memory */ 57 | extern void 58 | mem_prim_move32(uint32_t *dest, const uint32_t *src, uint32_t length); 59 | 60 | 61 | /* set bytes */ 62 | extern void 63 | mem_prim_set(void *dest, uint32_t dmax, uint8_t value); 64 | 65 | /* set uint16_ts */ 66 | extern void 67 | mem_prim_set16(uint16_t *dest, uint32_t dmax, uint16_t value); 68 | 69 | /* set uint32_ts */ 70 | extern void 71 | mem_prim_set32(uint32_t *dest, uint32_t dmax, uint32_t value); 72 | 73 | 74 | #endif /* __MEM_PRIMITIVES_LIB_H__ */ 75 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/safeclib/safe_mem_constraint.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * safe_mem_constraint.h 3 | * 4 | * October 2008, Bo Berry 5 | * 6 | * Copyright (c) 2008, 2009 by Cisco Systems, Inc. 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #ifndef __SAFE_MEM_CONSTRAINT_H__ 33 | #define __SAFE_MEM_CONSTRAINT_H__ 34 | 35 | #include "safeclib_private.h" 36 | 37 | /* 38 | * Function used by the libraries to invoke the registered 39 | * runtime-constraint handler. Always needed. 40 | */ 41 | extern void invoke_safe_mem_constraint_handler( 42 | const char *msg, 43 | void *ptr, 44 | errno_t error); 45 | 46 | #endif /* __SAFE_MEM_CONSTRAINT_H__ */ 47 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/safeclib/safeclib_private.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * safeclib_private.h - Internal library references 3 | * 4 | * 2012, Jonathan Toppins 5 | * 6 | * Copyright (c) 2012, 2013 by Cisco Systems, Inc 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #ifndef __SAFECLIB_PRIVATE_H__ 33 | #define __SAFECLIB_PRIVATE_H__ 34 | 35 | #include "citus_config.h" 36 | #ifdef __KERNEL__ 37 | /* linux kernel environment */ 38 | 39 | #include 40 | #include 41 | #include 42 | 43 | #define RCNEGATE(x) ( -(x) ) 44 | 45 | #define slprintf(...) printk(KERN_EMERG __VA_ARGS__) 46 | #define slabort() 47 | #ifdef DEBUG 48 | #define sldebug_printf(...) printk(KERN_DEBUG __VA_ARGS__) 49 | #endif 50 | 51 | #else /* !__KERNEL__ */ 52 | 53 | #if HAVE_CONFIG_H 54 | #include "config.h" 55 | #endif 56 | 57 | #include 58 | #ifdef STDC_HEADERS 59 | # include 60 | # include 61 | # include 62 | #else 63 | # ifdef HAVE_STDLIB_H 64 | # include 65 | # endif 66 | #endif 67 | #ifdef HAVE_STRING_H 68 | # if !defined STDC_HEADERS && defined HAVE_MEMORY_H 69 | # include 70 | # endif 71 | # include 72 | #endif 73 | #ifdef HAVE_LIMITS_H 74 | # include 75 | #endif 76 | 77 | #define EXPORT_SYMBOL(sym) 78 | #define RCNEGATE(x) (x) 79 | 80 | #define slprintf(...) fprintf(stderr, __VA_ARGS__) 81 | #define slabort() abort() 82 | #ifdef DEBUG 83 | #define sldebug_printf(...) printf(__VA_ARGS__) 84 | #endif 85 | 86 | #endif /* __KERNEL__ */ 87 | 88 | #ifndef sldebug_printf 89 | #define sldebug_printf(...) 90 | #endif 91 | 92 | #include "safe_lib.h" 93 | 94 | #endif /* __SAFECLIB_PRIVATE_H__ */ 95 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--10.0-1--10.0-2.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.0-1--10.0-2.sql 2 | 3 | -- grant read access for columnar metadata tables to unprivileged user 4 | GRANT USAGE ON SCHEMA columnar TO PUBLIC; 5 | GRANT SELECT ON ALL tables IN SCHEMA columnar TO PUBLIC ; 6 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--10.0-2--10.0-3.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.0-2--10.0-3.sql 2 | 3 | -- placeholder, no changes -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--10.0-3--10.1-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.0-3--10.1-1.sql 2 | 3 | -- Drop foreign keys between columnar metadata tables. 4 | -- Postgres assigns different names to those foreign keys in PG11, so act accordingly. 5 | DO $proc$ 6 | BEGIN 7 | IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN 8 | EXECUTE $$ 9 | ALTER TABLE columnar.chunk DROP CONSTRAINT chunk_storage_id_stripe_num_chunk_group_num_fkey; 10 | ALTER TABLE columnar.chunk_group DROP CONSTRAINT chunk_group_storage_id_stripe_num_fkey; 11 | $$; 12 | ELSE 13 | EXECUTE $$ 14 | ALTER TABLE columnar.chunk DROP CONSTRAINT chunk_storage_id_fkey; 15 | ALTER TABLE columnar.chunk_group DROP CONSTRAINT chunk_group_storage_id_fkey; 16 | $$; 17 | END IF; 18 | END$proc$; 19 | 20 | -- since we dropped pg11 support, we don't need to worry about missing 21 | -- columnar objects when upgrading postgres 22 | DROP FUNCTION columnar.columnar_ensure_objects_exist(); 23 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--10.1-1--10.2-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.1-1--10.2-1.sql 2 | 3 | -- For a proper mapping between tid & (stripe, row_num), add a new column to 4 | -- columnar.stripe and define a BTREE index on this column. 5 | -- Also include storage_id column for per-relation scans. 6 | ALTER TABLE columnar.stripe ADD COLUMN first_row_number bigint; 7 | CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number); 8 | 9 | -- Populate first_row_number column of columnar.stripe table. 10 | -- 11 | -- For simplicity, we calculate MAX(row_count) value across all the stripes 12 | -- of all the columanar tables and then use it to populate first_row_number 13 | -- column. This would introduce some gaps however we are okay with that since 14 | -- it's already the case with regular INSERT/COPY's. 15 | DO $$ 16 | DECLARE 17 | max_row_count bigint; 18 | -- this should be equal to columnar_storage.h/COLUMNAR_FIRST_ROW_NUMBER 19 | COLUMNAR_FIRST_ROW_NUMBER constant bigint := 1; 20 | BEGIN 21 | SELECT MAX(row_count) INTO max_row_count FROM columnar.stripe; 22 | UPDATE columnar.stripe SET first_row_number = COLUMNAR_FIRST_ROW_NUMBER + 23 | (stripe_num - 1) * max_row_count; 24 | END; 25 | $$; 26 | 27 | #include "udfs/upgrade_columnar_storage/10.2-1.sql" 28 | #include "udfs/downgrade_columnar_storage/10.2-1.sql" 29 | 30 | -- upgrade storage for all columnar relations 31 | SELECT columnar.upgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a 32 | WHERE c.relam = a.oid AND amname = 'columnar'; 33 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--10.2-1--10.2-2.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.2-1--10.2-2.sql 2 | 3 | -- revoke read access for columnar.chunk from unprivileged 4 | -- user as it contains chunk min/max values 5 | REVOKE SELECT ON columnar.chunk FROM PUBLIC; 6 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--10.2-2--10.2-3.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.2-2--10.2-3.sql 2 | 3 | -- Since stripe_first_row_number_idx is required to scan a columnar table, we 4 | -- need to make sure that it is created before doing anything with columnar 5 | -- tables during pg upgrades. 6 | -- 7 | -- However, a plain btree index is not a dependency of a table, so pg_upgrade 8 | -- cannot guarantee that stripe_first_row_number_idx gets created when 9 | -- creating columnar.stripe, unless we make it a unique "constraint". 10 | -- 11 | -- To do that, drop stripe_first_row_number_idx and create a unique 12 | -- constraint with the same name to keep the code change at minimum. 13 | DROP INDEX columnar.stripe_first_row_number_idx; 14 | ALTER TABLE columnar.stripe ADD CONSTRAINT stripe_first_row_number_idx 15 | UNIQUE (storage_id, first_row_number); 16 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--10.2-3--10.2-4.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.2-3--10.2-4.sql 2 | 3 | #include "udfs/columnar_ensure_am_depends_catalog/10.2-4.sql" 4 | 5 | SELECT columnar.columnar_ensure_am_depends_catalog(); 6 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--10.2-4--11.1-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.2.4--11.1-1.sql 2 | 3 | -- empty placeholder to allow standalone installation -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-1--11.1-2.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-1--11.2-1.sql 2 | 3 | #include "udfs/alter_table_set_access_method/11.1-2.sql" 4 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-10--11.1-11.sql: -------------------------------------------------------------------------------- 1 | #include "udfs/alter_table_set_access_method/11.1-11.sql" -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-11--11.1-12.sql: -------------------------------------------------------------------------------- 1 | #include "udfs/alter_table_set_access_method/11.1-12.sql" -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-2--11.1-3.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-1--11.2-1.sql 2 | 3 | #include "udfs/alter_columnar_table_set/11.1-3.sql" 4 | #include "udfs/alter_columnar_table_reset/11.1-3.sql" 5 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-4--11.1-5.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-4--11.1-5.sql 2 | 3 | #include "udfs/alter_table_set_access_method/11.1-5.sql" 4 | 5 | SET search_path TO columnar; 6 | 7 | CREATE SEQUENCE row_mask_seq START WITH 1 INCREMENT BY 1; 8 | 9 | CREATE TABLE row_mask ( 10 | id BIGINT NOT NULL, 11 | storage_id BIGINT NOT NULL, 12 | stripe_id BIGINT NOT NULL, 13 | chunk_id INT NOT NULL, 14 | start_row_number BIGINT NOT NULL, 15 | end_row_number BIGINT NOT NULL, 16 | deleted_rows INT NOT NULL, 17 | mask BYTEA, 18 | PRIMARY KEY (id, storage_id, start_row_number, end_row_number) 19 | ) WITH (user_catalog_table = true); 20 | 21 | ALTER TABLE columnar.chunk_group ADD COLUMN deleted_rows BIGINT NOT NULL DEFAULT 0; 22 | 23 | ALTER TABLE columnar.row_mask ADD CONSTRAINT row_mask_stripe_unique 24 | UNIQUE (storage_id, stripe_id, start_row_number); 25 | 26 | ALTER TABLE columnar.row_mask ADD CONSTRAINT row_mask_chunk_unique 27 | UNIQUE (storage_id, stripe_id, chunk_id, start_row_number); 28 | 29 | REVOKE SELECT ON columnar.row_mask FROM PUBLIC; 30 | 31 | COMMENT ON TABLE row_mask IS 'Columnar chunk mask metadata'; 32 | 33 | #include "udfs/create_table_row_mask/11.1-5.sql" 34 | 35 | -- find all columnar tables and create empty row mask for them 36 | 37 | SELECT columnar.create_table_row_mask(oid) FROM pg_class 38 | WHERE 39 | pg_class.relam = (SELECT oid FROM pg_am WHERE amname = 'columnar') 40 | AND 41 | pg_class.oid IN 42 | ( 43 | SELECT (quote_ident(pg_tables.schemaname) || '.' || 44 | quote_ident(pg_tables.tablename))::regclass::oid 45 | FROM pg_tables 46 | WHERE pg_tables.schemaname <> 'information_schema' AND 47 | pg_tables.schemaname <> 'pg_catalog' AND 48 | pg_tables.schemaname <> 'columnar' 49 | ); 50 | 51 | 52 | RESET search_path; -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-5--11.1-6.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-5--11.1-6.sql 2 | 3 | #include "udfs/vacuum/11.1-6.sql" 4 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-6--11.1-7.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-6--11.1-7.sql 2 | 3 | #include "udfs/vacuum/11.1-7.sql" 4 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-7--11.1-8.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-6--11.1-7.sql 2 | 3 | #include "udfs/alter_table_set_access_method/11.1-8.sql" -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--11.1-9--11.1-10.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION columnar.vacuum(REGCLASS, INT); 2 | DROP FUNCTION columnar.stats( 3 | IN regclass, 4 | OUT stripeId bigint, 5 | OUT fileOffset bigint, 6 | OUT rowCount integer, 7 | OUT deletedRows integer, 8 | OUT chunkCount integer, 9 | OUT dataLength integer 10 | ); 11 | DROP FUNCTION columnar.vacuum_full(NAME, REAL, INT); 12 | 13 | #include "udfs/vacuum/11.1-10.sql" 14 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/columnar--9.5-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--9.5-1.sql 2 | 3 | -- empty placeholder to allow standalone installation -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.0-1--9.5-1.sql 2 | 3 | SET search_path TO columnar; 4 | 5 | DO $proc$ 6 | BEGIN 7 | 8 | IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN 9 | EXECUTE $$ 10 | DROP FUNCTION pg_catalog.alter_columnar_table_reset( 11 | table_name regclass, 12 | chunk_group_row_limit bool, 13 | stripe_row_limit bool, 14 | compression bool, 15 | compression_level bool); 16 | 17 | DROP FUNCTION pg_catalog.alter_columnar_table_set( 18 | table_name regclass, 19 | chunk_group_row_limit int, 20 | stripe_row_limit int, 21 | compression name, 22 | compression_level int); 23 | 24 | DROP ACCESS METHOD columnar; 25 | 26 | DROP FUNCTION columnar_handler(internal); 27 | 28 | $$; 29 | END IF; 30 | END$proc$; 31 | 32 | DROP TABLE chunk; 33 | DROP TABLE chunk_group; 34 | DROP TABLE stripe; 35 | DROP TABLE options; 36 | DROP SEQUENCE storageid_seq; 37 | 38 | DROP FUNCTION columnar.columnar_ensure_objects_exist(); 39 | 40 | RESET search_path; 41 | DROP SCHEMA columnar; 42 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--10.0-2--10.0-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.0-2--10.0-1.sql 2 | 3 | -- revoke read access for columnar metadata tables from unprivileged user 4 | REVOKE USAGE ON SCHEMA columnar FROM PUBLIC; 5 | REVOKE SELECT ON ALL tables IN SCHEMA columnar FROM PUBLIC; 6 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--10.1-1--10.0-3.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.1-1--10.0-3.sql 2 | 3 | -- define foreign keys between columnar metadata tables 4 | ALTER TABLE columnar.chunk 5 | ADD FOREIGN KEY (storage_id, stripe_num, chunk_group_num) 6 | REFERENCES columnar.chunk_group(storage_id, stripe_num, chunk_group_num) ON DELETE CASCADE; 7 | 8 | ALTER TABLE columnar.chunk_group 9 | ADD FOREIGN KEY (storage_id, stripe_num) 10 | REFERENCES columnar.stripe(storage_id, stripe_num) ON DELETE CASCADE; 11 | 12 | -- define columnar_ensure_objects_exist again 13 | #include "../udfs/columnar_ensure_objects_exist/10.0-1.sql" 14 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--10.2-1--10.1-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.2-1--10.1-1.sql 2 | 3 | -- downgrade storage for all columnar relations 4 | SELECT columnar.downgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a 5 | WHERE c.relam = a.oid AND amname = 'columnar'; 6 | 7 | DROP FUNCTION columnar.upgrade_columnar_storage(regclass); 8 | DROP FUNCTION columnar.downgrade_columnar_storage(regclass); 9 | 10 | -- drop "first_row_number" column and the index defined on it 11 | DROP INDEX columnar.stripe_first_row_number_idx; 12 | ALTER TABLE columnar.stripe DROP COLUMN first_row_number; 13 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--10.2-2--10.2-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.2-2--10.2-1.sql 2 | 3 | -- grant read access for columnar.chunk to unprivileged user 4 | GRANT SELECT ON columnar.chunk TO PUBLIC; 5 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--10.2-3--10.2-2.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.2-3--10.2-2.sql 2 | 3 | ALTER TABLE columnar.stripe DROP CONSTRAINT stripe_first_row_number_idx; 4 | CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number); 5 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--10.2-4--10.2-3.sql: -------------------------------------------------------------------------------- 1 | -- columnar--10.2-4--10.2-3.sql 2 | 3 | DROP FUNCTION columnar.columnar_ensure_am_depends_catalog(); 4 | 5 | -- Note that we intentionally do not delete pg_depend records that we inserted 6 | -- via columnar--10.2-3--10.2-4.sql (by using columnar_ensure_am_depends_catalog). 7 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--11.1-1--10.2-4.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-1--10.2-4.sql 2 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--11.1-2--11.1-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-2--11.1-1.sql 2 | 3 | DROP FUNCTION columnar.alter_table_set_access_method(); -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--11.1-3--11.1-2.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-3--11.1-2.sql 2 | 3 | #include "udfs/alter_columnar_table_set/11.1-2.sql" 4 | #include "udfs/alter_columnar_table_reset/11.1-2.sql" 5 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/downgrades/columnar--11.1-5--11.1-4.sql: -------------------------------------------------------------------------------- 1 | -- columnar--11.1-4--11.1-4.sql 2 | 3 | DROP FUNCTION columnar.create_table_row_mask(); 4 | 5 | DROP TABLE columnar.row_mask; 6 | 7 | DROP SEQUENCE columnar.row_mask_seq; -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( 2 | table_name regclass, 3 | chunk_group_row_limit bool DEFAULT false, 4 | stripe_row_limit bool DEFAULT false, 5 | compression bool DEFAULT false, 6 | compression_level bool DEFAULT false) 7 | RETURNS void 8 | LANGUAGE C 9 | AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; 10 | 11 | COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( 12 | table_name regclass, 13 | chunk_group_row_limit bool, 14 | stripe_row_limit bool, 15 | compression bool, 16 | compression_level bool) 17 | IS 'reset on or more options on a columnar table to the system defaults'; 18 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/alter_columnar_table_reset/11.1-2.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS columnar.alter_columnar_table_reset; 2 | 3 | CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset( 4 | table_name regclass, 5 | chunk_group_row_limit bool DEFAULT false, 6 | stripe_row_limit bool DEFAULT false, 7 | compression bool DEFAULT false, 8 | compression_level bool DEFAULT false) 9 | RETURNS void 10 | LANGUAGE C 11 | AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; 12 | 13 | COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset( 14 | table_name regclass, 15 | chunk_group_row_limit bool, 16 | stripe_row_limit bool, 17 | compression bool, 18 | compression_level bool) 19 | IS 'reset on or more options on a columnar table to the system defaults'; 20 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/alter_columnar_table_reset/11.1-3.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS pg_catalog.alter_columnar_table_reset; 2 | 3 | CREATE OR REPLACE FUNCTION columnar.alter_columnar_table_reset( 4 | table_name regclass, 5 | chunk_group_row_limit bool DEFAULT false, 6 | stripe_row_limit bool DEFAULT false, 7 | compression bool DEFAULT false, 8 | compression_level bool DEFAULT false) 9 | RETURNS void 10 | LANGUAGE C 11 | AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; 12 | 13 | COMMENT ON FUNCTION columnar.alter_columnar_table_reset( 14 | table_name regclass, 15 | chunk_group_row_limit bool, 16 | stripe_row_limit bool, 17 | compression bool, 18 | compression_level bool) 19 | IS 'reset on or more options on a columnar table to the system defaults'; 20 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.alter_columnar_table_reset( 2 | table_name regclass, 3 | chunk_group_row_limit bool DEFAULT false, 4 | stripe_row_limit bool DEFAULT false, 5 | compression bool DEFAULT false, 6 | compression_level bool DEFAULT false) 7 | RETURNS void 8 | LANGUAGE C 9 | AS 'MODULE_PATHNAME', 'alter_columnar_table_reset'; 10 | 11 | COMMENT ON FUNCTION columnar.alter_columnar_table_reset( 12 | table_name regclass, 13 | chunk_group_row_limit bool, 14 | stripe_row_limit bool, 15 | compression bool, 16 | compression_level bool) 17 | IS 'reset on or more options on a columnar table to the system defaults'; 18 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( 2 | table_name regclass, 3 | chunk_group_row_limit int DEFAULT NULL, 4 | stripe_row_limit int DEFAULT NULL, 5 | compression name DEFAULT null, 6 | compression_level int DEFAULT NULL) 7 | RETURNS void 8 | LANGUAGE C 9 | AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; 10 | 11 | COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( 12 | table_name regclass, 13 | chunk_group_row_limit int, 14 | stripe_row_limit int, 15 | compression name, 16 | compression_level int) 17 | IS 'set one or more options on a columnar table, when set to NULL no change is made'; 18 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/alter_columnar_table_set/11.1-2.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS columnar.alter_columnar_table_set; 2 | 3 | CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set( 4 | table_name regclass, 5 | chunk_group_row_limit int DEFAULT NULL, 6 | stripe_row_limit int DEFAULT NULL, 7 | compression name DEFAULT null, 8 | compression_level int DEFAULT NULL) 9 | RETURNS void 10 | LANGUAGE C 11 | AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; 12 | 13 | COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set( 14 | table_name regclass, 15 | chunk_group_row_limit int, 16 | stripe_row_limit int, 17 | compression name, 18 | compression_level int) 19 | IS 'set one or more options on a columnar table, when set to NULL no change is made'; 20 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/alter_columnar_table_set/11.1-3.sql: -------------------------------------------------------------------------------- 1 | DROP FUNCTION IF EXISTS pg_catalog.alter_columnar_table_set; 2 | 3 | CREATE OR REPLACE FUNCTION columnar.alter_columnar_table_set( 4 | table_name regclass, 5 | chunk_group_row_limit int DEFAULT NULL, 6 | stripe_row_limit int DEFAULT NULL, 7 | compression name DEFAULT null, 8 | compression_level int DEFAULT NULL) 9 | RETURNS void 10 | LANGUAGE C 11 | AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; 12 | 13 | COMMENT ON FUNCTION columnar.alter_columnar_table_set( 14 | table_name regclass, 15 | chunk_group_row_limit int, 16 | stripe_row_limit int, 17 | compression name, 18 | compression_level int) 19 | IS 'set one or more options on a columnar table, when set to NULL no change is made'; 20 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.alter_columnar_table_set( 2 | table_name regclass, 3 | chunk_group_row_limit int DEFAULT NULL, 4 | stripe_row_limit int DEFAULT NULL, 5 | compression name DEFAULT null, 6 | compression_level int DEFAULT NULL) 7 | RETURNS void 8 | LANGUAGE C 9 | AS 'MODULE_PATHNAME', 'alter_columnar_table_set'; 10 | 11 | COMMENT ON FUNCTION columnar.alter_columnar_table_set( 12 | table_name regclass, 13 | chunk_group_row_limit int, 14 | stripe_row_limit int, 15 | compression name, 16 | compression_level int) 17 | IS 'set one or more options on a columnar table, when set to NULL no change is made'; 18 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/10.2-4.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.columnar_ensure_am_depends_catalog() 2 | RETURNS void 3 | LANGUAGE plpgsql 4 | SET search_path = pg_catalog 5 | AS $func$ 6 | BEGIN 7 | INSERT INTO pg_depend 8 | SELECT -- Define a dependency edge from "columnar table access method" .. 9 | 'pg_am'::regclass::oid as classid, 10 | (select oid from pg_am where amname = 'columnar') as objid, 11 | 0 as objsubid, 12 | -- ... to each object that is registered to pg_class and that lives 13 | -- in "columnar" schema. That contains catalog tables, indexes 14 | -- created on them and the sequences created in "columnar" schema. 15 | -- 16 | -- Given the possibility of user might have created their own objects 17 | -- in columnar schema, we explicitly specify list of objects that we 18 | -- are interested in. 19 | 'pg_class'::regclass::oid as refclassid, 20 | columnar_schema_members.relname::regclass::oid as refobjid, 21 | 0 as refobjsubid, 22 | 'n' as deptype 23 | FROM (VALUES ('columnar.chunk'), 24 | ('columnar.chunk_group'), 25 | ('columnar.chunk_group_pkey'), 26 | ('columnar.chunk_pkey'), 27 | ('columnar.options'), 28 | ('columnar.options_pkey'), 29 | ('columnar.storageid_seq'), 30 | ('columnar.stripe'), 31 | ('columnar.stripe_first_row_number_idx'), 32 | ('columnar.stripe_pkey') 33 | ) columnar_schema_members(relname) 34 | -- Avoid inserting duplicate entries into pg_depend. 35 | EXCEPT TABLE pg_depend; 36 | END; 37 | $func$; 38 | COMMENT ON FUNCTION columnar.columnar_ensure_am_depends_catalog() 39 | IS 'internal function responsible for creating dependencies from columnar ' 40 | 'table access method to the rel objects in columnar schema'; 41 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/latest.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.columnar_ensure_am_depends_catalog() 2 | RETURNS void 3 | LANGUAGE plpgsql 4 | SET search_path = pg_catalog 5 | AS $func$ 6 | BEGIN 7 | INSERT INTO pg_depend 8 | SELECT -- Define a dependency edge from "columnar table access method" .. 9 | 'pg_am'::regclass::oid as classid, 10 | (select oid from pg_am where amname = 'columnar') as objid, 11 | 0 as objsubid, 12 | -- ... to each object that is registered to pg_class and that lives 13 | -- in "columnar" schema. That contains catalog tables, indexes 14 | -- created on them and the sequences created in "columnar" schema. 15 | -- 16 | -- Given the possibility of user might have created their own objects 17 | -- in columnar schema, we explicitly specify list of objects that we 18 | -- are interested in. 19 | 'pg_class'::regclass::oid as refclassid, 20 | columnar_schema_members.relname::regclass::oid as refobjid, 21 | 0 as refobjsubid, 22 | 'n' as deptype 23 | FROM (VALUES ('columnar.chunk'), 24 | ('columnar.chunk_group'), 25 | ('columnar.chunk_group_pkey'), 26 | ('columnar.chunk_pkey'), 27 | ('columnar.options'), 28 | ('columnar.options_pkey'), 29 | ('columnar.storageid_seq'), 30 | ('columnar.stripe'), 31 | ('columnar.stripe_first_row_number_idx'), 32 | ('columnar.stripe_pkey') 33 | ) columnar_schema_members(relname) 34 | -- Avoid inserting duplicate entries into pg_depend. 35 | EXCEPT TABLE pg_depend; 36 | END; 37 | $func$; 38 | COMMENT ON FUNCTION columnar.columnar_ensure_am_depends_catalog() 39 | IS 'internal function responsible for creating dependencies from columnar ' 40 | 'table access method to the rel objects in columnar schema'; 41 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/10.0-1.sql: -------------------------------------------------------------------------------- 1 | -- columnar.columnar_ensure_objects_exist is an internal helper function to create 2 | -- missing objects, anything related to the table access methods. 3 | -- Since the API for table access methods is only available in PG12 we can't create these 4 | -- objects when Citus is installed in PG11. Once citus is installed on PG11 the user can 5 | -- upgrade their database to PG12. Now they require the table access method objects that 6 | -- we couldn't create before. 7 | -- This internal function is called from `citus_finish_pg_upgrade` which the user is 8 | -- required to call after a PG major upgrade. 9 | CREATE OR REPLACE FUNCTION columnar.columnar_ensure_objects_exist() 10 | RETURNS void 11 | LANGUAGE plpgsql 12 | SET search_path = pg_catalog 13 | AS $ceoe$ 14 | BEGIN 15 | 16 | -- when postgres is version 12 or above we need to create the tableam. If the tableam 17 | -- exist we assume all objects have been created. 18 | IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN 19 | IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN 20 | 21 | #include "../columnar_handler/10.0-1.sql" 22 | 23 | #include "../alter_columnar_table_set/10.0-1.sql" 24 | 25 | #include "../alter_columnar_table_reset/10.0-1.sql" 26 | 27 | -- add the missing objects to the extension 28 | ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler(internal); 29 | ALTER EXTENSION citus ADD ACCESS METHOD columnar; 30 | ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set( 31 | table_name regclass, 32 | chunk_group_row_limit int, 33 | stripe_row_limit int, 34 | compression name, 35 | compression_level int); 36 | ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset( 37 | table_name regclass, 38 | chunk_group_row_limit bool, 39 | stripe_row_limit bool, 40 | compression bool, 41 | compression_level bool); 42 | 43 | END IF; 44 | END IF; 45 | END; 46 | $ceoe$; 47 | 48 | COMMENT ON FUNCTION columnar.columnar_ensure_objects_exist() 49 | IS 'internal function to be called by pg_catalog.citus_finish_pg_upgrade responsible for creating the columnar objects'; 50 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/latest.sql: -------------------------------------------------------------------------------- 1 | -- columnar.columnar_ensure_objects_exist is an internal helper function to create 2 | -- missing objects, anything related to the table access methods. 3 | -- Since the API for table access methods is only available in PG12 we can't create these 4 | -- objects when Citus is installed in PG11. Once citus is installed on PG11 the user can 5 | -- upgrade their database to PG12. Now they require the table access method objects that 6 | -- we couldn't create before. 7 | -- This internal function is called from `citus_finish_pg_upgrade` which the user is 8 | -- required to call after a PG major upgrade. 9 | CREATE OR REPLACE FUNCTION columnar.columnar_ensure_objects_exist() 10 | RETURNS void 11 | LANGUAGE plpgsql 12 | SET search_path = pg_catalog 13 | AS $ceoe$ 14 | BEGIN 15 | 16 | -- when postgres is version 12 or above we need to create the tableam. If the tableam 17 | -- exist we assume all objects have been created. 18 | IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN 19 | IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN 20 | 21 | #include "../columnar_handler/10.0-1.sql" 22 | 23 | #include "../alter_columnar_table_set/10.0-1.sql" 24 | 25 | #include "../alter_columnar_table_reset/10.0-1.sql" 26 | 27 | -- add the missing objects to the extension 28 | ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler(internal); 29 | ALTER EXTENSION citus ADD ACCESS METHOD columnar; 30 | ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set( 31 | table_name regclass, 32 | chunk_group_row_limit int, 33 | stripe_row_limit int, 34 | compression name, 35 | compression_level int); 36 | ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset( 37 | table_name regclass, 38 | chunk_group_row_limit bool, 39 | stripe_row_limit bool, 40 | compression bool, 41 | compression_level bool); 42 | 43 | END IF; 44 | END IF; 45 | END; 46 | $ceoe$; 47 | 48 | COMMENT ON FUNCTION columnar.columnar_ensure_objects_exist() 49 | IS 'internal function to be called by pg_catalog.citus_finish_pg_upgrade responsible for creating the columnar objects'; 50 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/columnar_handler/10.0-1.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal) 2 | RETURNS table_am_handler 3 | LANGUAGE C 4 | AS 'MODULE_PATHNAME', 'columnar_handler'; 5 | 6 | COMMENT ON FUNCTION columnar.columnar_handler(internal) 7 | IS 'internal function returning the handler for columnar tables'; 8 | 9 | -- postgres 11.8 does not support the syntax for table am, also it is seemingly trying 10 | -- to parse the upgrade file and erroring on unknown syntax. 11 | -- normally this section would not execute on postgres 11 anyway. To trick it to pass on 12 | -- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid 13 | -- syntax on 11.8 and will execute correctly in 12 14 | DO $create_table_am$ 15 | BEGIN 16 | EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler'; 17 | END $create_table_am$; 18 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/columnar_handler/latest.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal) 2 | RETURNS table_am_handler 3 | LANGUAGE C 4 | AS 'MODULE_PATHNAME', 'columnar_handler'; 5 | 6 | COMMENT ON FUNCTION columnar.columnar_handler(internal) 7 | IS 'internal function returning the handler for columnar tables'; 8 | 9 | -- postgres 11.8 does not support the syntax for table am, also it is seemingly trying 10 | -- to parse the upgrade file and erroring on unknown syntax. 11 | -- normally this section would not execute on postgres 11 anyway. To trick it to pass on 12 | -- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid 13 | -- syntax on 11.8 and will execute correctly in 12 14 | DO $create_table_am$ 15 | BEGIN 16 | EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler'; 17 | END $create_table_am$; 18 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/create_table_row_mask/11.1-5.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.create_table_row_mask( 2 | table_name regclass) 3 | RETURNS bool 4 | LANGUAGE C 5 | AS 'MODULE_PATHNAME', 'create_table_row_mask'; 6 | 7 | COMMENT ON FUNCTION columnar.create_table_row_mask( 8 | table_name regclass) 9 | IS 'Create empty row mask for table'; -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/create_table_row_mask/latest.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.create_table_row_mask( 2 | table_name regclass) 3 | RETURNS bool 4 | LANGUAGE C 5 | AS 'MODULE_PATHNAME', 'create_table_row_mask'; 6 | 7 | COMMENT ON FUNCTION columnar.create_table_row_mask( 8 | table_name regclass) 9 | IS 'Create empty row mask for table'; -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/downgrade_columnar_storage/10.2-1.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.downgrade_columnar_storage(rel regclass) 2 | RETURNS VOID 3 | STRICT 4 | LANGUAGE c AS 'MODULE_PATHNAME', $$downgrade_columnar_storage$$; 5 | 6 | COMMENT ON FUNCTION columnar.downgrade_columnar_storage(regclass) 7 | IS 'function to downgrade the columnar storage, if necessary'; 8 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/downgrade_columnar_storage/latest.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.downgrade_columnar_storage(rel regclass) 2 | RETURNS VOID 3 | STRICT 4 | LANGUAGE c AS 'MODULE_PATHNAME', $$downgrade_columnar_storage$$; 5 | 6 | COMMENT ON FUNCTION columnar.downgrade_columnar_storage(regclass) 7 | IS 'function to downgrade the columnar storage, if necessary'; 8 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/upgrade_columnar_storage/10.2-1.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.upgrade_columnar_storage(rel regclass) 2 | RETURNS VOID 3 | STRICT 4 | LANGUAGE c AS 'MODULE_PATHNAME', $$upgrade_columnar_storage$$; 5 | 6 | COMMENT ON FUNCTION columnar.upgrade_columnar_storage(regclass) 7 | IS 'function to upgrade the columnar storage, if necessary'; 8 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/upgrade_columnar_storage/latest.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.upgrade_columnar_storage(rel regclass) 2 | RETURNS VOID 3 | STRICT 4 | LANGUAGE c AS 'MODULE_PATHNAME', $$upgrade_columnar_storage$$; 5 | 6 | COMMENT ON FUNCTION columnar.upgrade_columnar_storage(regclass) 7 | IS 'function to upgrade the columnar storage, if necessary'; 8 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/vacuum/11.1-10.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar._vacuum_internal(rel REGCLASS, stripe_count INT) 2 | RETURNS INT 3 | LANGUAGE c AS 'MODULE_PATHNAME', $$vacuum_columnar_table$$; 4 | 5 | COMMENT ON FUNCTION columnar._vacuum_internal(REGCLASS, INT) 6 | IS 'vacuum columnar table internal function'; 7 | 8 | CREATE OR REPLACE FUNCTION columnar.vacuum(tablename REGCLASS, stripe_count INT DEFAULT 0) 9 | RETURNS INT AS $$ 10 | DECLARE 11 | count INT; 12 | stripes INT; 13 | BEGIN 14 | count := 1; 15 | stripes := 0; 16 | 17 | WHILE count > 0 AND (stripe_count = 0 OR stripes < stripe_count) LOOP 18 | SELECT columnar._vacuum_internal(tablename, stripe_count) INTO count; 19 | stripes := stripes + count; 20 | END LOOP; 21 | 22 | return stripes; 23 | END; 24 | $$ LANGUAGE plpgsql; 25 | 26 | COMMENT ON FUNCTION columnar.vacuum(REGCLASS, INT) 27 | IS 'vacuum columnar table function'; 28 | 29 | CREATE OR REPLACE FUNCTION columnar.stats( 30 | IN regclass, 31 | OUT stripeId bigint, 32 | OUT fileOffset bigint, 33 | OUT rowCount integer, 34 | OUT deletedRows integer, 35 | OUT chunkCount integer, 36 | OUT dataLength integer 37 | ) RETURNS SETOF record 38 | LANGUAGE c 39 | AS 'MODULE_PATHNAME', $$columnar_stats$$; 40 | 41 | COMMENT ON FUNCTION columnar.stats(regclass) 42 | IS 'columnar stripe statistics'; 43 | 44 | CREATE OR REPLACE FUNCTION columnar.vacuum_full(schema NAME DEFAULT 'public', sleep_time REAL DEFAULT .1, stripe_count INT DEFAULT 25) 45 | RETURNS VOID AS $$ 46 | DECLARE 47 | tables REGCLASS[]; 48 | tablename REGCLASS; 49 | finished BOOL; 50 | count INT; 51 | BEGIN 52 | SELECT ARRAY_AGG(c.relname) INTO tables 53 | FROM pg_catalog.pg_class c 54 | LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace 55 | LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam 56 | WHERE c.relkind = 'r' 57 | AND n.nspname <> 'pg_catalog' 58 | AND n.nspname !~ '^pg_toast' 59 | AND n.nspname <> 'information_schema' 60 | AND pg_catalog.pg_table_is_visible(c.oid) 61 | AND am.amname = 'columnar' 62 | AND n.nspname = schema 63 | ORDER BY 1; 64 | 65 | FOREACH tablename IN ARRAY tables 66 | LOOP 67 | finished := 'f'; 68 | count := 1; 69 | WHILE count > 0 LOOP 70 | SELECT columnar.vacuum(tablename, stripe_count) INTO count; 71 | PERFORM pg_sleep(sleep_time); 72 | END LOOP; 73 | END LOOP; 74 | END; 75 | $$ LANGUAGE plpgsql; 76 | 77 | COMMENT ON FUNCTION columnar.vacuum_full(name, real, int) 78 | IS 'vacuum columnar schema in full incrementally'; 79 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/vacuum/11.1-6.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar.vacuum(rel regclass, optional_count int DEFAULT 0) 2 | RETURNS INT 3 | LANGUAGE c AS 'MODULE_PATHNAME', $$vacuum_columnar_table$$; 4 | 5 | COMMENT ON FUNCTION columnar.vacuum(regclass, int) 6 | IS 'vacuum columnar table'; 7 | 8 | CREATE OR REPLACE FUNCTION columnar.stats( 9 | IN regclass, 10 | OUT stripeId bigint, 11 | OUT fileOffset bigint, 12 | OUT rowCount integer, 13 | OUT deletedRows integer, 14 | OUT chunkCount integer, 15 | OUT dataLength integer 16 | ) RETURNS SETOF record 17 | LANGUAGE c 18 | AS 'MODULE_PATHNAME', $$columnar_stats$$; 19 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/vacuum/11.1-7.sql: -------------------------------------------------------------------------------- 1 | COMMENT ON FUNCTION columnar.stats(regclass) 2 | IS 'columnar stripe statistics'; 3 | 4 | CREATE OR REPLACE FUNCTION columnar.vacuum_full(schema NAME DEFAULT 'public', sleep_time REAL DEFAULT .1, stripe_count INT DEFAULT 25) 5 | RETURNS VOID AS $$ 6 | DECLARE 7 | tables REGCLASS[]; 8 | tablename REGCLASS; 9 | finished BOOL; 10 | count INT; 11 | BEGIN 12 | SELECT ARRAY_AGG(c.relname) INTO tables 13 | FROM pg_catalog.pg_class c 14 | LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace 15 | LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam 16 | WHERE c.relkind = 'r' 17 | AND n.nspname <> 'pg_catalog' 18 | AND n.nspname !~ '^pg_toast' 19 | AND n.nspname <> 'information_schema' 20 | AND pg_catalog.pg_table_is_visible(c.oid) 21 | AND am.amname = 'columnar' 22 | AND n.nspname = schema 23 | ORDER BY 1; 24 | 25 | FOREACH tablename IN ARRAY tables 26 | LOOP 27 | finished := 'f'; 28 | count := 1; 29 | WHILE count > 0 LOOP 30 | SELECT columnar.vacuum(tablename, stripe_count) INTO count; 31 | PERFORM pg_sleep(sleep_time); 32 | END LOOP; 33 | END LOOP; 34 | END; 35 | $$ LANGUAGE plpgsql; 36 | 37 | COMMENT ON FUNCTION columnar.vacuum_full(name, real, int) 38 | IS 'vacuum columnar schema in full incrementally'; 39 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/sql/udfs/vacuum/latest.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION columnar._vacuum_internal(rel REGCLASS, stripe_count INT) 2 | RETURNS INT 3 | LANGUAGE c AS 'MODULE_PATHNAME', $$vacuum_columnar_table$$; 4 | 5 | COMMENT ON FUNCTION columnar._vacuum_internal(REGCLASS, INT) 6 | IS 'vacuum columnar table internal function'; 7 | 8 | CREATE OR REPLACE FUNCTION columnar.vacuum(tablename REGCLASS, stripe_count INT DEFAULT 0) 9 | RETURNS INT AS $$ 10 | DECLARE 11 | count INT; 12 | stripes INT; 13 | BEGIN 14 | count := 1; 15 | stripes := 0; 16 | 17 | WHILE count > 0 AND (stripe_count = 0 OR stripes < stripe_count) LOOP 18 | SELECT columnar._vacuum_internal(tablename, stripe_count) INTO count; 19 | stripes := stripes + count; 20 | END LOOP; 21 | 22 | return stripes; 23 | END; 24 | $$ LANGUAGE plpgsql; 25 | 26 | COMMENT ON FUNCTION columnar.vacuum(REGCLASS, INT) 27 | IS 'vacuum columnar table function'; 28 | 29 | CREATE OR REPLACE FUNCTION columnar.stats( 30 | IN regclass, 31 | OUT stripeId bigint, 32 | OUT fileOffset bigint, 33 | OUT rowCount integer, 34 | OUT deletedRows integer, 35 | OUT chunkCount integer, 36 | OUT dataLength integer 37 | ) RETURNS SETOF record 38 | LANGUAGE c 39 | AS 'MODULE_PATHNAME', $$columnar_stats$$; 40 | 41 | COMMENT ON FUNCTION columnar.stats(regclass) 42 | IS 'columnar stripe statistics'; 43 | 44 | CREATE OR REPLACE FUNCTION columnar.vacuum_full(schema NAME DEFAULT 'public', sleep_time REAL DEFAULT .1, stripe_count INT DEFAULT 25) 45 | RETURNS VOID AS $$ 46 | DECLARE 47 | tables REGCLASS[]; 48 | tablename REGCLASS; 49 | finished BOOL; 50 | count INT; 51 | BEGIN 52 | SELECT ARRAY_AGG(c.relname) INTO tables 53 | FROM pg_catalog.pg_class c 54 | LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace 55 | LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam 56 | WHERE c.relkind = 'r' 57 | AND n.nspname <> 'pg_catalog' 58 | AND n.nspname !~ '^pg_toast' 59 | AND n.nspname <> 'information_schema' 60 | AND pg_catalog.pg_table_is_visible(c.oid) 61 | AND am.amname = 'columnar' 62 | AND n.nspname = schema 63 | ORDER BY 1; 64 | 65 | FOREACH tablename IN ARRAY tables 66 | LOOP 67 | finished := 'f'; 68 | count := 1; 69 | WHILE count > 0 LOOP 70 | SELECT columnar.vacuum(tablename, stripe_count) INTO count; 71 | PERFORM pg_sleep(sleep_time); 72 | END LOOP; 73 | END LOOP; 74 | END; 75 | $$ LANGUAGE plpgsql; 76 | 77 | COMMENT ON FUNCTION columnar.vacuum_full(name, real, int) 78 | IS 'vacuum columnar schema in full incrementally'; 79 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/vectorization/types/date.c: -------------------------------------------------------------------------------- 1 | 2 | #include "postgres.h" 3 | 4 | #include "nodes/execnodes.h" 5 | #include "utils/date.h" 6 | 7 | #include "columnar/vectorization/types/types.h" 8 | 9 | // date (int32) 10 | BUILD_CMP_OPERATOR_INT(date_, DateADT, DateADT) 11 | 12 | // time (int64) 13 | BUILD_CMP_OPERATOR_INT(time_, TimeADT, TimeADT) 14 | 15 | // timestamp (int64) 16 | BUILD_CMP_OPERATOR_INT(timestamp_, Timestamp, Timestamp) 17 | 18 | // timestamptz (int64) 19 | BUILD_CMP_OPERATOR_INT(timestamptz_, TimestampTz, TimestampTz) 20 | -------------------------------------------------------------------------------- /columnar/src/backend/columnar/vectorization/types/int.c: -------------------------------------------------------------------------------- 1 | 2 | #include "postgres.h" 3 | #include "common/int.h" 4 | 5 | #include "fmgr.h" 6 | #include "nodes/execnodes.h" 7 | 8 | #include "columnar/vectorization/types/types.h" 9 | 10 | // char 11 | BUILD_CMP_OPERATOR_INT( char, char, char) 12 | 13 | // int2 14 | BUILD_CMP_OPERATOR_INT( int2, int16, int16) 15 | BUILD_CMP_OPERATOR_INT(int24, int16, int32) 16 | BUILD_CMP_OPERATOR_INT(int28, int16, int64) 17 | 18 | // int4 19 | BUILD_CMP_OPERATOR_INT( int4, int32, int32) 20 | BUILD_CMP_OPERATOR_INT(int42, int32, int16) 21 | BUILD_CMP_OPERATOR_INT(int48, int32, int64) 22 | 23 | // int8 24 | BUILD_CMP_OPERATOR_INT( int8, int64, int64) 25 | BUILD_CMP_OPERATOR_INT(int82, int64, int16) 26 | BUILD_CMP_OPERATOR_INT(int84, int64, int32) 27 | -------------------------------------------------------------------------------- /columnar/src/include/.gitignore: -------------------------------------------------------------------------------- 1 | /stamp-h 2 | /stamp-ext-h 3 | /citus_config.h 4 | /citus_config.h.in~ 5 | /citus_version.h 6 | /citus_version.h.in~ 7 | -------------------------------------------------------------------------------- /columnar/src/include/citus_version.h.in: -------------------------------------------------------------------------------- 1 | /* This file is created manually */ 2 | 3 | /* Citus full name as a string */ 4 | #undef CITUS_NAME 5 | 6 | /* Citus edition as a string */ 7 | #undef CITUS_EDITION 8 | 9 | /* Extension version expected by this Citus build */ 10 | #undef CITUS_EXTENSIONVERSION 11 | 12 | /* Citus major version as a string */ 13 | #undef CITUS_MAJORVERSION 14 | 15 | /* Citus version as a string */ 16 | #undef CITUS_VERSION 17 | 18 | /* Citus version as a number */ 19 | #undef CITUS_VERSION_NUM 20 | 21 | /* A string containing the version number, platform, and C compiler */ 22 | #undef CITUS_VERSION_STR 23 | 24 | /* Define to 1 if you have the `curl' library (-lcurl). */ 25 | #undef HAVE_LIBCURL 26 | 27 | /* Define to 1 if you have the `liblz4' library (-llz4). */ 28 | #undef HAVE_CITUS_LIBLZ4 29 | 30 | /* Define to 1 if you have the `libzstd' library (-lzstd). */ 31 | #undef HAVE_LIBZSTD 32 | 33 | /* Base URL for statistics collection and update checks */ 34 | #undef REPORTS_BASE_URL 35 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/columnar_compression.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_compression.h 4 | * 5 | * Type and function declarations for compression methods. 6 | * 7 | * Copyright (c) Citus Data, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | #ifndef COLUMNAR_COMPRESSION_H 13 | #define COLUMNAR_COMPRESSION_H 14 | 15 | /* Enumaration for columnar table's compression method */ 16 | typedef enum 17 | { 18 | COMPRESSION_TYPE_INVALID = -1, 19 | COMPRESSION_NONE = 0, 20 | COMPRESSION_PG_LZ = 1, 21 | COMPRESSION_LZ4 = 2, 22 | COMPRESSION_ZSTD = 3, 23 | 24 | COMPRESSION_COUNT 25 | } CompressionType; 26 | 27 | extern bool CompressBuffer(StringInfo inputBuffer, 28 | StringInfo outputBuffer, 29 | CompressionType compressionType, 30 | int compressionLevel); 31 | extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressionType, 32 | uint64 decompressedSize); 33 | 34 | #endif /* COLUMNAR_COMPRESSION_H */ 35 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/columnar_customscan.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_customscan.h 4 | * 5 | * Forward declarations of functions to hookup the custom scan feature of 6 | * columnar. 7 | * 8 | * $Id$ 9 | * 10 | *------------------------------------------------------------------------- 11 | */ 12 | 13 | #ifndef COLUMNAR_CUSTOMSCAN_H 14 | #define COLUMNAR_CUSTOMSCAN_H 15 | 16 | #include "nodes/extensible.h" 17 | 18 | /* Flag to indicate is vectorized aggregate used in execution */ 19 | #define CUSTOM_SCAN_VECTORIZED_AGGREGATE 1 20 | 21 | extern void columnar_customscan_init(void); 22 | extern const CustomScanMethods * columnar_customscan_methods(void); 23 | extern Bitmapset * ColumnarAttrNeeded(ScanState *ss, List *customList); 24 | 25 | #endif /* COLUMNAR_CUSTOMSCAN_H */ 26 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/columnar_indexscan.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_indexscan.h 4 | * Custom scan method for index 5 | * 6 | * IDENTIFICATION 7 | * src/backend/columnar/columnar_indexscan.c 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | 13 | #ifndef COLUMNAR_INDEXSCAN_H 14 | #define COLUMNAR_INDEXSCAN_H 15 | 16 | #include "postgres.h" 17 | 18 | #include "nodes/execnodes.h" 19 | 20 | typedef struct ColumnarIndexScanState 21 | { 22 | CustomScanState css; 23 | IndexScanState *indexscan_state; 24 | } ColumnarIndexScanState; 25 | 26 | extern CustomScan * columnar_create_indexscan_node(void); 27 | extern void columnar_register_indexscan_node(void); 28 | 29 | #endif -------------------------------------------------------------------------------- /columnar/src/include/columnar/columnar_metadata.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_metadata.h 4 | * 5 | * Type and function declarations for Columnar metadata. 6 | * 7 | * Copyright (c) Citus Data, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | #ifndef COLUMNAR_METADATA_H 13 | #define COLUMNAR_METADATA_H 14 | 15 | #include "access/sdir.h" 16 | 17 | /* 18 | * StripeMetadata represents information about a stripe. This information is 19 | * stored in the metadata table "columnar.stripe". 20 | */ 21 | typedef struct StripeMetadata 22 | { 23 | uint64 fileOffset; 24 | uint64 dataLength; 25 | uint32 columnCount; 26 | uint32 chunkCount; 27 | uint32 chunkGroupRowCount; 28 | uint64 rowCount; 29 | uint64 id; 30 | uint64 firstRowNumber; 31 | 32 | /* see StripeWriteState */ 33 | bool aborted; 34 | 35 | /* 36 | * If write operation is in-progress (i.e. StripeWriteState returned 37 | * STRIPE_WRITE_IN_PROGRESS), then insertedByCurrentXact is used to 38 | * distinguish whether it's being written by current transaction or 39 | * not. 40 | */ 41 | bool insertedByCurrentXact; 42 | } StripeMetadata; 43 | 44 | /* 45 | * EmptyStripeReservation represents information for an empty stripe 46 | * reservation. 47 | */ 48 | typedef struct EmptyStripeReservation 49 | { 50 | uint64 stripeId; 51 | uint64 stripeFirstRowNumber; 52 | } EmptyStripeReservation; 53 | 54 | extern List * StripesForRelfilenode(RelFileLocator relfilelocator, ScanDirection scanDirection); 55 | extern uint32 DeletedRowsForStripe(RelFileLocator relfilelocator, 56 | uint32 chunkCount, 57 | uint64 stripeId); 58 | extern Size DecompressedLengthForStripe(RelFileLocator relfilelocator, uint64 stripeId); 59 | extern void ColumnarStorageUpdateIfNeeded(Relation rel, bool isUpgrade); 60 | extern StripeMetadata * RewriteStripeMetadataRowWithNewValues(Relation rel, uint64 stripeId, 61 | uint64 sizeBytes, uint64 fileOffset, uint64 rowCount, uint64 chunkCount); 62 | 63 | #endif /* COLUMNAR_METADATA_H */ 64 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/columnar_storage.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_storage.h 4 | * 5 | * Type and function declarations for storage of columnar data in blocks. 6 | * 7 | * Copyright (c) Citus Data, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | #ifndef COLUMNAR_STORAGE_H 13 | #define COLUMNAR_STORAGE_H 14 | 15 | #include "postgres.h" 16 | 17 | #include "storage/smgr.h" 18 | #include "utils/rel.h" 19 | 20 | #include "columnar/columnar_tableam.h" 21 | 22 | 23 | #define COLUMNAR_INVALID_ROW_NUMBER ((uint64) 0) 24 | #define COLUMNAR_FIRST_ROW_NUMBER ((uint64) 1) 25 | #define COLUMNAR_MAX_ROW_NUMBER ((uint64) \ 26 | (COLUMNAR_FIRST_ROW_NUMBER + \ 27 | VALID_ITEMPOINTER_OFFSETS * \ 28 | VALID_BLOCKNUMBERS)) 29 | 30 | 31 | /* 32 | * Logical offsets never fall on the first two physical pages. See 33 | * comments in columnar_storage.c. 34 | */ 35 | #define ColumnarInvalidLogicalOffset 0 36 | #define ColumnarFirstLogicalOffset ((BLCKSZ - SizeOfPageHeaderData) * 2) 37 | #define ColumnarLogicalOffsetIsValid(X) ((X) >= ColumnarFirstLogicalOffset) 38 | 39 | 40 | extern void ColumnarStorageInit(SMgrRelation srel, uint64 storageId); 41 | extern bool ColumnarStorageIsCurrent(Relation rel); 42 | extern void ColumnarStorageUpdateCurrent(Relation rel, bool upgrade, 43 | uint64 reservedStripeId, 44 | uint64 reservedRowNumber, 45 | uint64 reservedOffset); 46 | 47 | extern uint64 ColumnarStorageGetVersionMajor(Relation rel, bool force); 48 | extern uint64 ColumnarStorageGetVersionMinor(Relation rel, bool force); 49 | extern uint64 ColumnarStorageGetStorageId(Relation rel, bool force); 50 | extern uint64 ColumnarStorageGetReservedStripeId(Relation rel, bool force); 51 | extern uint64 ColumnarStorageGetReservedRowNumber(Relation rel, bool force); 52 | extern uint64 ColumnarStorageGetReservedOffset(Relation rel, bool force); 53 | 54 | extern uint64 ColumnarStorageReserveData(Relation rel, uint64 amount); 55 | extern uint64 ColumnarStorageReserveRowNumber(Relation rel, uint64 nrows); 56 | extern uint64 ColumnarStorageReserveStripeId(Relation rel); 57 | 58 | extern void ColumnarStorageRead(Relation rel, uint64 logicalOffset, 59 | char *data, uint32 amount); 60 | extern void ColumnarStorageWrite(Relation rel, uint64 logicalOffset, 61 | char *data, uint32 amount); 62 | extern bool ColumnarStorageTruncate(Relation rel, uint64 newDataReservation); 63 | 64 | #endif /* COLUMNAR_STORAGE_H */ 65 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/columnar_tableam.h: -------------------------------------------------------------------------------- 1 | #ifndef COLUMNAR_TABLEAM_H 2 | #define COLUMNAR_TABLEAM_H 3 | #include "citus_version.h" 4 | 5 | #include "postgres.h" 6 | #include "fmgr.h" 7 | #include "access/tableam.h" 8 | #include "access/skey.h" 9 | #include "nodes/bitmapset.h" 10 | #include "access/heapam.h" 11 | #include "catalog/indexing.h" 12 | #include "utils/acl.h" 13 | 14 | #include "columnar/columnar.h" 15 | 16 | /* 17 | * Number of valid ItemPointer Offset's for "row number" <> "ItemPointer" 18 | * mapping. 19 | * 20 | * Postgres has some asserts calling either ItemPointerIsValid or 21 | * OffsetNumberIsValid. That constraints itemPointer.offsetNumber 22 | * for columnar tables to the following interval: 23 | * [FirstOffsetNumber, MaxOffsetNumber]. 24 | * 25 | * However, bitmap scan logic assumes that itemPointer.offsetNumber cannot 26 | * be larger than MaxHeapTuplesPerPage (see tbm_add_tuples). 27 | * 28 | * For this reason, we restrict itemPointer.offsetNumber 29 | * to the following interval: [FirstOffsetNumber, MaxHeapTuplesPerPage]. 30 | */ 31 | #define VALID_ITEMPOINTER_OFFSETS \ 32 | ((uint64) (MaxHeapTuplesPerPage - FirstOffsetNumber + 1)) 33 | 34 | /* 35 | * Number of valid ItemPointer BlockNumber's for "row number" <> "ItemPointer" 36 | * mapping. 37 | * 38 | * Similar to VALID_ITEMPOINTER_OFFSETS, due to asserts around 39 | * itemPointer.blockNumber, we can only use values upto and including 40 | * MaxBlockNumber. 41 | * Note that postgres doesn't restrict blockNumber to a lower boundary. 42 | * 43 | * For this reason, we restrict itemPointer.blockNumber 44 | * to the following interval: [0, MaxBlockNumber]. 45 | */ 46 | #define VALID_BLOCKNUMBERS ((uint64) (MaxBlockNumber + 1)) 47 | 48 | 49 | struct ColumnarScanDescData; 50 | typedef struct ColumnarScanDescData *ColumnarScanDesc; 51 | 52 | 53 | const TableAmRoutine * GetColumnarTableAmRoutine(void); 54 | extern void columnar_tableam_init(void); 55 | extern TableScanDesc columnar_beginscan_extended(Relation relation, Snapshot snapshot, 56 | int nkeys, ScanKey key, 57 | ParallelTableScanDesc parallel_scan, 58 | uint32 flags, Bitmapset *attr_needed, 59 | List *scanQual, 60 | ParallelColumnarScan parallelColumnarScan, 61 | bool returnVectorResult); 62 | extern IndexFetchTableData * columnar_index_fetch_begin_extended(Relation rel, 63 | Bitmapset *attr_neededs); 64 | extern int64 ColumnarScanChunkGroupsFiltered(ColumnarScanDesc columnarScanDesc); 65 | extern bool ColumnarSupportsIndexAM(char *indexAMName); 66 | extern bool IsColumnarTableAmTable(Oid relationId); 67 | 68 | 69 | #endif /* COLUMNAR_TABLEAM_H */ 70 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/columnar_version_compat.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_version_compat.h 4 | * 5 | * Compatibility macros for writing code agnostic to PostgreSQL versions 6 | * 7 | * Copyright (c) Citus Data, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | #ifndef COLUMNAR_COMPAT_H 13 | #define COLUMNAR_COMPAT_H 14 | 15 | #include "pg_version_constants.h" 16 | 17 | #if PG_VERSION_NUM >= PG_VERSION_14 18 | #define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \ 19 | ColumnarProcessUtility(a, b, c, d, e, f, g, h) 20 | #define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \ 21 | PrevProcessUtilityHook(a, b, c, d, e, f, g, h) 22 | #define GetOldestNonRemovableTransactionId_compat(a, b) \ 23 | GetOldestNonRemovableTransactionId(a) 24 | #define ExecSimpleRelationInsert_compat(a, b, c) \ 25 | ExecSimpleRelationInsert(a, b, c) 26 | #define index_insert_compat(a, b, c, d, e, f, g, h) \ 27 | index_insert(a, b, c, d, e, f, g, h) 28 | #else 29 | #define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \ 30 | ColumnarProcessUtility(a, b, d, e, f, g, h) 31 | #define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \ 32 | PrevProcessUtilityHook(a, b, d, e, f, g, h) 33 | #define GetOldestNonRemovableTransactionId_compat(a, b) GetOldestXmin(a, b) 34 | #define ExecSimpleRelationInsert_compat(a, b, c) \ 35 | ExecSimpleRelationInsert(b, c) 36 | #define index_insert_compat(a, b, c, d, e, f, g, h) \ 37 | index_insert(a, b, c, d, e, f, h) 38 | #endif 39 | 40 | #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE 41 | 42 | #define ExplainPropertyLong(qlabel, value, es) \ 43 | ExplainPropertyInteger(qlabel, NULL, value, es) 44 | 45 | #if PG_VERSION_NUM < 130000 46 | #define detoast_attr(X) heap_tuple_untoast_attr(X) 47 | #endif 48 | 49 | #endif /* COLUMNAR_COMPAT_H */ 50 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/columnar_write_state_row_mask.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_write_state_row_mask.h 4 | * 5 | * Declaration for Row Mask Entry 6 | * 7 | * Copyright (c) Hydra, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | #ifndef COLUMNAR_WRITE_STATE_ROW_MASK_H 13 | #define COLUMNAR_WRITE_STATE_ROW_MASK_H 14 | 15 | struct RowMaskWriteStateEntry 16 | { 17 | uint64 id; 18 | uint64 storageId; 19 | uint64 stripeId; 20 | uint32 chunkId; 21 | int64 startRowNumber; 22 | int64 endRowNumber; 23 | int32 deletedRows; 24 | bytea *mask; 25 | }; 26 | 27 | #endif -------------------------------------------------------------------------------- /columnar/src/include/columnar/vectorization/columnar_vector_execution.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_vector_execution.h 4 | * 5 | * Vectorization execution function 6 | * 7 | * Copyright (c) Hydra, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | #ifndef COLUMNAR_VECTOR_EXECUTION_H 13 | #define COLUMNAR_VECTOR_EXECUTION_H 14 | 15 | #include "executor/tuptable.h" 16 | #include "nodes/pg_list.h" 17 | #include "nodes/primnodes.h" 18 | 19 | extern bool CheckOpExprArgumentRules(List *args); 20 | extern bool GetVectorizedProcedureOid(Oid procedureOid, Oid *vectorizedProcedureOid); 21 | extern List * CreateVectorizedExprList(List *exprList); 22 | extern List * ConstructVectorizedQualList(TupleTableSlot *slot, List *vectorizedQual); 23 | extern bool * ExecuteVectorizedQual(TupleTableSlot *slot, 24 | List *vectorizedQualList, 25 | BoolExprType boolType, 26 | ExprContext *econtext); 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/vectorization/columnar_vector_types.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_vector_types.h 4 | * 5 | * Structures used in vectorization execution 6 | * 7 | * Copyright (c) Hydra, Inc. 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | #ifndef COLUMNAR_VECTOR_TYPES_H 13 | #define COLUMNAR_VECTOR_TYPES_H 14 | 15 | #include "postgres.h" 16 | #include "fmgr.h" 17 | 18 | #include "executor/tuptable.h" 19 | #include "nodes/bitmapset.h" 20 | #include "nodes/primnodes.h" 21 | 22 | /* DEFAULT_CHUNK_ROW_COUNT */ 23 | #define COLUMNAR_VECTOR_COLUMN_SIZE 10000 24 | 25 | typedef struct VectorTupleTableSlot 26 | { 27 | /* TupleTableSlot structure */ 28 | TupleTableSlot tts; 29 | /* How many tuples does this slot contain */ 30 | uint32 dimension; 31 | /* Keep array to represent filtered tuples */ 32 | bool keep[COLUMNAR_VECTOR_COLUMN_SIZE]; 33 | /* Row Number */ 34 | uint64 rowNumber[COLUMNAR_VECTOR_COLUMN_SIZE]; 35 | } VectorTupleTableSlot; 36 | 37 | extern TupleTableSlot * CreateVectorTupleTableSlot(TupleDesc tupleDesc); 38 | 39 | typedef struct VectorColumn 40 | { 41 | uint32 dimension; 42 | uint16 columnTypeLen; 43 | bool columnIsVal; 44 | Datum *value; 45 | bool isnull[COLUMNAR_VECTOR_COLUMN_SIZE]; 46 | uint64 *rowNumber; 47 | } VectorColumn; 48 | 49 | extern VectorColumn * BuildVectorColumn(int16 columnDimension, 50 | int16 columnTypeLen, 51 | bool columnIsVal, 52 | uint64 *rowNumber); 53 | extern void ExtractTupleFromVectorSlot(TupleTableSlot *out, 54 | VectorTupleTableSlot *vectorSlot, 55 | int32 index, 56 | List *attrNeededList); 57 | extern void WriteTupleToVectorSlot(TupleTableSlot *in, 58 | VectorTupleTableSlot *vectorSlot, 59 | int32 index); 60 | extern void CleanupVectorSlot(VectorTupleTableSlot *vectorSlot); 61 | 62 | typedef enum VectorQualType 63 | { 64 | VECTOR_QUAL_BOOL_EXPR, 65 | VECTOR_QUAL_EXPR 66 | } VectorQualTypeEnum; 67 | 68 | 69 | typedef enum VectorArgType 70 | { 71 | VECTOR_FN_ARG_CONSTANT, 72 | VECTOR_FN_ARG_VAR 73 | } VectorFnArgTypeEnum; 74 | 75 | 76 | typedef struct VectorFnArgument 77 | { 78 | VectorFnArgTypeEnum type; 79 | Datum arg; 80 | } VectorFnArgument; 81 | 82 | 83 | typedef struct VectorQual 84 | { 85 | VectorQualTypeEnum vectorQualType; 86 | union 87 | { 88 | struct 89 | { 90 | FmgrInfo *fmgrInfo; 91 | FunctionCallInfo fcInfo; 92 | VectorFnArgument *vectorFnArguments; 93 | } expr; 94 | struct 95 | { 96 | BoolExprType boolExprType; 97 | List *vectorQualExprList; 98 | } boolExpr; 99 | } u; 100 | } VectorQual; 101 | 102 | #endif 103 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/vectorization/nodes/columnar_aggregator_node.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * columnar_aggregator_node.h 4 | * Custom scan method for aggregation 5 | * 6 | * IDENTIFICATION 7 | * src/backend/columnar/vectorization/nodes/columnar_aggregator_node.c 8 | * 9 | *------------------------------------------------------------------------- 10 | */ 11 | 12 | 13 | #ifndef COLUMNAR_AGGEREGATOR_NODE_H 14 | #define COLUMNAR_AGGEREGATOR_NODE_H 15 | 16 | #include "postgres.h" 17 | 18 | #include "nodes/execnodes.h" 19 | 20 | typedef struct VectorAggState 21 | { 22 | CustomScanState css; 23 | AggState *aggstate; 24 | } VectorAggState; 25 | 26 | extern CustomScan *columnar_create_aggregator_node(void); 27 | extern void columnar_register_aggregator_node(void); 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /columnar/src/include/columnar/vectorization/types/numeric.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * numeric.h 4 | * 5 | * IDENTIFICATION 6 | * src/backend/columnar/vectorization/types/numeric.c 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | 11 | #ifndef VECTORIZATION_TYPE_INT128_H 12 | #define VECTORIZATION_TYPE_INT128_H 13 | 14 | #include "postgres.h" 15 | 16 | #include "utils/numeric.h" 17 | extern Numeric int128_to_numeric(int128 val); 18 | 19 | #endif -------------------------------------------------------------------------------- /columnar/src/include/pg_version_constants.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------------- 2 | * 3 | * pg_version_constants.h 4 | * pg version related constants. 5 | * 6 | * Copyright (c) Citus Data, Inc. 7 | * 8 | *------------------------------------------------------------------------- 9 | */ 10 | 11 | #ifndef PG_VERSION_CONSTANTS 12 | #define PG_VERSION_CONSTANTS 13 | 14 | #define PG_VERSION_12 120000 15 | #define PG_VERSION_13 130000 16 | #define PG_VERSION_14 140000 17 | #define PG_VERSION_15 150000 18 | #define PG_VERSION_16 160000 19 | 20 | #endif /* PG_VERSION_CONSTANTS */ 21 | -------------------------------------------------------------------------------- /columnar/src/test/regress/.gitignore: -------------------------------------------------------------------------------- 1 | # Local binaries 2 | /pg_regress 3 | 4 | # Generated subdirectories 5 | /tmp_check/ 6 | /results/ 7 | /log/ 8 | 9 | # Regression test output 10 | /regression.diffs 11 | /regression.output 12 | /regression.out 13 | 14 | # python 15 | *.pyc 16 | 17 | # core dumps 18 | core 19 | -------------------------------------------------------------------------------- /columnar/src/test/regress/Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for tests of the Citus extension 2 | 3 | citus_subdir = src/test/regress 4 | citus_top_builddir = ../../.. 5 | 6 | include $(citus_top_builddir)/Makefile.global 7 | 8 | PG_VERSION_NUM := $(shell cat `$(PG_CONFIG) --includedir-server`/pg_config*.h \ 9 | | perl -ne 'print $$1 and exit if /PG_VERSION_NUM\s+(\d+)/') 10 | 11 | # Test input and expected files. These are created by pg_regress itself, so we 12 | # don't have a rule to create them. We do need rules to clean them however. 13 | input_files := $(patsubst $(citus_abs_srcdir)/input/%.source,sql/%.sql, $(wildcard $(citus_abs_srcdir)/input/*.source)) 14 | output_files := $(patsubst $(citus_abs_srcdir)/output/%.source,expected/%.out, $(wildcard $(citus_abs_srcdir)/output/*.source)) 15 | 16 | check-all: check-regression-columnar 17 | 18 | check-regression-columnar: 19 | ifeq ($(shell test $(PG_VERSION_NUM) -gt 149999; echo $$?),0) 20 | # postgres 15 removed the whole concept of input/output files 21 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < input/columnar_copyto.source > sql/columnar_copyto.sql 22 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < input/columnar_data_types.source > sql/columnar_data_types.sql 23 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < input/columnar_load.source > sql/columnar_load.sql 24 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < output/columnar_copyto.source > expected/columnar_copyto.out 25 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < output/columnar_data_types.source > expected/columnar_data_types.out 26 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < output/columnar_load.source > expected/columnar_load.out 27 | endif 28 | TEST_DIR=$(PWD) $(pg_regress_check) \ 29 | --temp-config columnar_regression.conf \ 30 | --load-extension=columnar \ 31 | --schedule=$(citus_abs_srcdir)/columnar_schedule 32 | 33 | clean-regression: 34 | rm -fr $(citus_abs_srcdir)/tmp_check 35 | rm -fr $(citus_abs_srcdir)/log 36 | rm -fr $(citus_abs_srcdir)/results 37 | rm -f $(citus_abs_srcdir)/regression.diffs 38 | rm -f $(citus_abs_srcdir)/regression.out 39 | rm -f $(citus_abs_srcdir)/$(output_files) 40 | rm -fr $(citus_abs_srcdir)/$(input_files) 41 | 42 | clean: clean-regression 43 | -------------------------------------------------------------------------------- /columnar/src/test/regress/columnar_regression.conf: -------------------------------------------------------------------------------- 1 | # Columnar storage engine configuration 2 | 3 | shared_preload_libraries = 'columnar.so' 4 | log_temp_files = -1 -------------------------------------------------------------------------------- /columnar/src/test/regress/columnar_schedule: -------------------------------------------------------------------------------- 1 | test: columnar_test_helpers 2 | 3 | test: columnar_create 4 | test: columnar_load 5 | test: columnar_query columnar_first_row_number 6 | test: columnar_analyze 7 | test: columnar_data_types 8 | test: columnar_drop 9 | test: columnar_indexes 10 | test: columnar_fallback_scan columnar_paths 11 | #test: columnar_partitioning 12 | test: columnar_permissions 13 | test: columnar_empty 14 | test: columnar_insert 15 | test: columnar_update_delete 16 | test: columnar_cursor 17 | test: columnar_copyto 18 | test: columnar_alter 19 | test: columnar_alter_set_type 20 | test: columnar_lz4 columnar_zstd 21 | test: columnar_rollback 22 | test: columnar_truncate 23 | test: columnar_vacuum 24 | test: columnar_vacuum_udf 25 | test: columnar_clean 26 | test: columnar_types_without_comparison 27 | #test: columnar_chunk_filtering 28 | test: columnar_join 29 | test: columnar_trigger 30 | test: columnar_tableoptions 31 | test: columnar_recursive 32 | test: columnar_transactions 33 | test: columnar_matview 34 | #test: columnar_memory 35 | test: columnar_alter_table_set_access_method 36 | test: columnar_cache 37 | test: columnar_aggregates 38 | test: columnar_upsert 39 | test: columnar_customindex 40 | test: columnar_vectorization 41 | -------------------------------------------------------------------------------- /columnar/src/test/regress/data/array_types.csv: -------------------------------------------------------------------------------- 1 | "{1,2,3}","{1,2,3}","{a,b,c}" 2 | {},{},{} 3 | "{-2147483648,2147483647}","{-9223372036854775808,9223372036854775807}","{""""}" 4 | -------------------------------------------------------------------------------- /columnar/src/test/regress/data/contestants.1.csv: -------------------------------------------------------------------------------- 1 | a,1990-01-10,2090,97.1,XA ,{a} 2 | b,1990-11-01,2203,98.1,XA ,"{a,b}" 3 | c,1988-11-01,2907,99.4,XB ,"{w,y}" 4 | d,1985-05-05,2314,98.3,XB ,{} 5 | e,1995-05-05,2236,98.2,XC ,{a} 6 | -------------------------------------------------------------------------------- /columnar/src/test/regress/data/contestants.2.csv: -------------------------------------------------------------------------------- 1 | f,1983-04-02,3090,99.6,XD ,"{a,b,c,y}" 2 | g,1991-12-13,1803,85.1,XD ,"{a,c}" 3 | h,1987-10-26,2112,95.4,XD ,"{w,a}" 4 | -------------------------------------------------------------------------------- /columnar/src/test/regress/data/datetime_types.csv: -------------------------------------------------------------------------------- 1 | 2000-01-02 04:05:06,1999-01-08 14:05:06+02,2000-01-02,04:05:06,04:00:00 2 | 1970-01-01 00:00:00,infinity,-infinity,00:00:00,00:00:00 3 | -------------------------------------------------------------------------------- /columnar/src/test/regress/data/enum_and_composite_types.csv: -------------------------------------------------------------------------------- 1 | a,"(2,b)" 2 | b,"(3,c)" 3 | -------------------------------------------------------------------------------- /columnar/src/test/regress/data/null_values.csv: -------------------------------------------------------------------------------- 1 | ,{NULL},"(,)" 2 | ,, 3 | -------------------------------------------------------------------------------- /columnar/src/test/regress/data/other_types.csv: -------------------------------------------------------------------------------- 1 | f,\xdeadbeef,$1.00,192.168.1.2,10101,a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11,"{""key"": ""value""}" 2 | t,\xcdb0,$1.50,127.0.0.1,"",a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11,[] 3 | -------------------------------------------------------------------------------- /columnar/src/test/regress/data/range_types.csv: -------------------------------------------------------------------------------- 1 | "[1,3)","[1,3)","[1,3)","[""2000-01-02 00:30:00"",""2010-02-03 12:30:00"")" 2 | empty,"[1,)","(,)",empty 3 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/.gitignore: -------------------------------------------------------------------------------- 1 | /columnar_copyto.out 2 | /columnar_data_types.out 3 | /columnar_load.out 4 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_alter_set_type.out: -------------------------------------------------------------------------------- 1 | -- 2 | -- Testing ALTER TABLE on columnar tables. 3 | -- 4 | CREATE TABLE test_alter_table (a int, b int, c text) USING columnar; 5 | WITH sample_data AS (VALUES 6 | (1, 2, '3'), 7 | (4, 5, '6') 8 | ) 9 | INSERT INTO test_alter_table SELECT * FROM sample_data; 10 | WITH sample_data AS (VALUES 11 | (5, 9, '11'), 12 | (12, 83, '93') 13 | ) 14 | INSERT INTO test_alter_table SELECT * FROM sample_data; 15 | ALTER TABLE test_alter_table ALTER COLUMN a TYPE jsonb USING row_to_json(row(a)); 16 | SELECT * FROM test_alter_table ORDER BY a; 17 | a | b | c 18 | ------------+----+---- 19 | {"f1": 1} | 2 | 3 20 | {"f1": 4} | 5 | 6 21 | {"f1": 5} | 9 | 11 22 | {"f1": 12} | 83 | 93 23 | (4 rows) 24 | 25 | ALTER TABLE test_alter_table ALTER COLUMN c TYPE int USING c::integer; 26 | SELECT sum(c) FROM test_alter_table; 27 | sum 28 | ----- 29 | 113 30 | (1 row) 31 | 32 | ALTER TABLE test_alter_table ALTER COLUMN b TYPE bigint; 33 | SELECT * FROM test_alter_table ORDER BY a; 34 | a | b | c 35 | ------------+----+---- 36 | {"f1": 1} | 2 | 3 37 | {"f1": 4} | 5 | 6 38 | {"f1": 5} | 9 | 11 39 | {"f1": 12} | 83 | 93 40 | (4 rows) 41 | 42 | ALTER TABLE test_alter_table ALTER COLUMN b TYPE float USING (b::float + 0.5); 43 | SELECT * FROM test_alter_table ORDER BY a; 44 | a | b | c 45 | ------------+------+---- 46 | {"f1": 1} | 2.5 | 3 47 | {"f1": 4} | 5.5 | 6 48 | {"f1": 5} | 9.5 | 11 49 | {"f1": 12} | 83.5 | 93 50 | (4 rows) 51 | 52 | DROP TABLE test_alter_table; 53 | -- Make sure that the correct table options are used when rewriting the table. 54 | -- This is reflected by the VACUUM VERBOSE output right after a rewrite showing 55 | -- that all chunks are compressed with the configured compression algorithm 56 | CREATE TABLE test(i int) USING columnar; 57 | SELECT columnar.alter_columnar_table_set('test', compression => 'lz4'); 58 | alter_columnar_table_set 59 | -------------------------- 60 | 61 | (1 row) 62 | 63 | INSERT INTO test VALUES(1); 64 | VACUUM VERBOSE test; 65 | INFO: statistics for "test": 66 | storage id: 10000000144 67 | total file size: 24576, total data size: 6 68 | compression rate: 0.83x 69 | total row count: 1, stripe count: 1, average rows per stripe: 1 70 | chunk count: 1, containing data for dropped columns: 0, lz4 compressed: 1 71 | 72 | ALTER TABLE test ALTER COLUMN i TYPE int8; 73 | VACUUM VERBOSE test; 74 | INFO: statistics for "test": 75 | storage id: 10000000145 76 | total file size: 24576, total data size: 10 77 | compression rate: 0.90x 78 | total row count: 1, stripe count: 1, average rows per stripe: 1 79 | chunk count: 1, containing data for dropped columns: 0, lz4 compressed: 1 80 | 81 | DROP TABLE test; 82 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_analyze.out: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test the ANALYZE command for columnar tables. 3 | -- 4 | -- ANALYZE uncompressed table 5 | ANALYZE contestant; 6 | SELECT count(*) FROM pg_stats WHERE tablename='contestant'; 7 | count 8 | ------- 9 | 6 10 | (1 row) 11 | 12 | -- ANALYZE compressed table 13 | ANALYZE contestant_compressed; 14 | SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed'; 15 | count 16 | ------- 17 | 6 18 | (1 row) 19 | 20 | -- ANALYZE a table with lots of data to trigget qsort in analyze.c 21 | CREATE TABLE test_analyze(a int, b text, c char) USING columnar; 22 | INSERT INTO test_analyze SELECT floor(i / 1000), floor(i / 10)::text, 4 FROM generate_series(1, 100000) i; 23 | INSERT INTO test_analyze SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 110000) i; 24 | ANALYZE test_analyze; 25 | DROP TABLE test_analyze; 26 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_clean.out: -------------------------------------------------------------------------------- 1 | DROP TABLE test_null_values; 2 | DROP TABLE test_other_types; 3 | DROP TABLE test_range_types; 4 | DROP TABLE test_enum_and_composite_types; 5 | DROP TYPE composite_type; 6 | DROP TYPE enum_type; 7 | DROP TABLE test_datetime_types; 8 | DROP TABLE test_array_types; 9 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_customindex.out: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test custom index: Test shows only that we are exchanging Index scan with CustomIndexScan while 3 | -- all other information is still the same. The feature is not visible through EXPLAIN because output 4 | -- will be same in both cases, but difference is noticeable how the columnar storage engine is read. 5 | -- With index scan ALL columns are requested from storage (this can be significant overhead in performance) 6 | -- while with CustomIndexScan we will request only columns that are needed - that are going to be used as output. 7 | -- Test also shows that we are not using CustomIndexScan with heap tables and only on columnar tables. 8 | -- 9 | SET columnar.enable_columnar_index_scan TO TRUE; 10 | CREATE TABLE t(a INT PRIMARY KEY, b INT, c TEXT) USING columnar; 11 | CREATE INDEX t_idx ON t USING btree(b); 12 | CREATE TABLE t_heap(a INT PRIMARY KEY, b INT, c TEXT); 13 | CREATE INDEX t_idx_heap ON t_heap USING btree(b); 14 | INSERT INTO t SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g; 15 | INSERT INTO t_heap SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g; 16 | -- make sure that we test index scan 17 | set columnar.enable_custom_scan TO 'off'; 18 | set enable_seqscan TO off; 19 | set seq_page_cost TO 10000000; 20 | EXPLAIN (VERBOSE) SELECT a FROM t WHERE b > 18 ORDER BY b LIMIT 10; 21 | QUERY PLAN 22 | --------------------------------------------------------------------------------- 23 | Limit (cost=0.30..0.56 rows=10 width=8) 24 | Output: a, b 25 | -> Custom Scan (ColumnarIndexScan) (cost=0.30..2586.30 rows=100000 width=8) 26 | Output: a, b 27 | ColumnarIndexScan using : t_idx 28 | Index Cond: (t.b > 18) 29 | (6 rows) 30 | 31 | EXPLAIN (VERBOSE) SELECT a FROM t_heap WHERE b > 18 ORDER BY b LIMIT 10; 32 | QUERY PLAN 33 | --------------------------------------------------------------------------------------------- 34 | Limit (cost=0.29..1.51 rows=10 width=8) 35 | Output: a, b 36 | -> Index Scan using t_idx_heap on public.t_heap (cost=0.29..9305.99 rows=76440 width=8) 37 | Output: a, b 38 | Index Cond: (t_heap.b > 18) 39 | (5 rows) 40 | 41 | DROP TABLE t; 42 | DROP TABLE t_heap; 43 | SET columnar.enable_custom_index_scan TO default; 44 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_customindex_1.out: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test custom index: Test shows only that we are exchanging Index scan with CustomIndexScan while 3 | -- all other information is still the same. The feature is not visible through EXPLAIN because output 4 | -- will be same in both cases, but difference is noticeable how the columnar storage engine is read. 5 | -- With index scan ALL columns are requested from storage (this can be significant overhead in performance) 6 | -- while with CustomIndexScan we will request only columns that are needed - that are going to be used as output. 7 | -- Test also shows that we are not using CustomIndexScan with heap tables and only on columnar tables. 8 | -- 9 | SET columnar.enable_columnar_index_scan TO TRUE; 10 | CREATE TABLE t(a INT PRIMARY KEY, b INT, c TEXT) USING columnar; 11 | CREATE INDEX t_idx ON t USING btree(b); 12 | CREATE TABLE t_heap(a INT PRIMARY KEY, b INT, c TEXT); 13 | CREATE INDEX t_idx_heap ON t_heap USING btree(b); 14 | INSERT INTO t SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g; 15 | INSERT INTO t_heap SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g; 16 | -- make sure that we test index scan 17 | set columnar.enable_custom_scan TO 'off'; 18 | set enable_seqscan TO off; 19 | set seq_page_cost TO 10000000; 20 | EXPLAIN (VERBOSE) SELECT a FROM t WHERE b > 18 ORDER BY b LIMIT 10; 21 | QUERY PLAN 22 | ------------------------------------------------------------------------------------ 23 | Limit (cost=0.30..0.56 rows=10 width=8) 24 | Output: a, b 25 | -> Index Scan using t_idx on public.t (cost=0.30..2586.30 rows=100000 width=8) 26 | Output: a, b 27 | Index Cond: (t.b > 18) 28 | (5 rows) 29 | 30 | EXPLAIN (VERBOSE) SELECT a FROM t_heap WHERE b > 18 ORDER BY b LIMIT 10; 31 | QUERY PLAN 32 | --------------------------------------------------------------------------------------------- 33 | Limit (cost=0.29..1.51 rows=10 width=8) 34 | Output: a, b 35 | -> Index Scan using t_idx_heap on public.t_heap (cost=0.29..9305.99 rows=76440 width=8) 36 | Output: a, b 37 | Index Cond: (t_heap.b > 18) 38 | (5 rows) 39 | 40 | DROP TABLE t; 41 | DROP TABLE t_heap; 42 | SET columnar.enable_custom_index_scan TO default; 43 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_drop.out: -------------------------------------------------------------------------------- 1 | -- 2 | -- Tests the different DROP commands for columnar tables. 3 | -- 4 | -- DROP TABL 5 | -- DROP SCHEMA 6 | -- DROP EXTENSION 7 | -- DROP DATABASE 8 | -- 9 | -- Note that travis does not create 10 | -- citus extension in default database (postgres). This has caused 11 | -- different behavior between travis tests and local tests. Thus 12 | -- 'postgres' directory is excluded from comparison to have the same result. 13 | -- store postgres database oid 14 | SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset 15 | SELECT count(distinct storage_id) AS columnar_stripes_before_drop FROM columnar.stripe \gset 16 | -- DROP columnar tables 17 | DROP TABLE contestant; 18 | DROP TABLE contestant_compressed; 19 | -- make sure DROP deletes metadata 20 | SELECT :columnar_stripes_before_drop - count(distinct storage_id) FROM columnar.stripe; 21 | ?column? 22 | ---------- 23 | 2 24 | (1 row) 25 | 26 | -- Create a columnar table under a schema and drop it. 27 | CREATE SCHEMA test_schema; 28 | CREATE TABLE test_schema.test_table(data int) USING columnar; 29 | INSERT INTO test_schema.test_table VALUES (1); 30 | SELECT count(*) AS columnar_stripes_before_drop FROM columnar.stripe \gset 31 | DROP SCHEMA test_schema CASCADE; 32 | NOTICE: drop cascades to table test_schema.test_table 33 | SELECT :columnar_stripes_before_drop - count(distinct storage_id) FROM columnar.stripe; 34 | ?column? 35 | ---------- 36 | 1 37 | (1 row) 38 | 39 | SELECT current_database() datname \gset 40 | CREATE DATABASE db_to_drop; 41 | \c db_to_drop 42 | CREATE EXTENSION columnar; 43 | SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset 44 | CREATE TABLE test_table(data int) USING columnar; 45 | DROP EXTENSION columnar CASCADE; 46 | NOTICE: drop cascades to table test_table 47 | -- test database drop 48 | CREATE EXTENSION columnar; 49 | SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset 50 | CREATE TABLE test_table(data int) USING columnar; 51 | \c :datname 52 | DROP DATABASE db_to_drop; 53 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_fallback_scan.out: -------------------------------------------------------------------------------- 1 | -- 2 | -- columnar_fallback_scan.sql 3 | -- 4 | -- Test columnar.enable_custom_scan = false, which will use an 5 | -- ordinary sequential scan. It won't benefit from projection pushdown 6 | -- or qual pushdown, but it should return the correct results. 7 | -- 8 | set columnar.enable_custom_scan = false; 9 | create table fallback_scan(i int) using columnar; 10 | -- large enough to test parallel_workers > 1 11 | select columnar.alter_columnar_table_set('fallback_scan', compression => 'none'); 12 | alter_columnar_table_set 13 | -------------------------- 14 | 15 | (1 row) 16 | 17 | insert into fallback_scan select generate_series(1,150000); 18 | vacuum analyze fallback_scan; 19 | select count(*), min(i), max(i), avg(i) from fallback_scan; 20 | count | min | max | avg 21 | --------+-----+--------+-------------------- 22 | 150000 | 1 | 150000 | 75000.500000000000 23 | (1 row) 24 | 25 | -- 26 | -- Negative test: try to force a parallel plan with at least two 27 | -- workers, but columnar should reject it and use a non-parallel scan. 28 | -- 29 | set min_parallel_table_scan_size = 1; 30 | set parallel_tuple_cost = 0; 31 | set max_parallel_workers = 4; 32 | set max_parallel_workers_per_gather = 4; 33 | explain (costs off) select count(*), min(i), max(i), avg(i) from fallback_scan; 34 | QUERY PLAN 35 | --------------------------------- 36 | Aggregate 37 | -> Seq Scan on fallback_scan 38 | (2 rows) 39 | 40 | select count(*), min(i), max(i), avg(i) from fallback_scan; 41 | count | min | max | avg 42 | --------+-----+--------+-------------------- 43 | 150000 | 1 | 150000 | 75000.500000000000 44 | (1 row) 45 | 46 | set min_parallel_table_scan_size to default; 47 | set parallel_tuple_cost to default; 48 | set max_parallel_workers to default; 49 | set max_parallel_workers_per_gather to default; 50 | set columnar.enable_custom_scan to default; 51 | drop table fallback_scan; 52 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_first_row_number.out: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA columnar_first_row_number; 2 | SET search_path tO columnar_first_row_number; 3 | CREATE TABLE col_table_1 (a int) USING columnar; 4 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 10) i; 5 | BEGIN; 6 | -- we don't use same first_row_number even if the xact is rollback'ed 7 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 11) i; 8 | ROLLBACK; 9 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 12) i; 10 | SELECT columnar.alter_columnar_table_set('col_table_1', stripe_row_limit => 1000); 11 | alter_columnar_table_set 12 | -------------------------- 13 | 14 | (1 row) 15 | 16 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 2350) i; 17 | SELECT row_count, first_row_number FROM columnar.stripe a 18 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass) 19 | ORDER BY stripe_num; 20 | row_count | first_row_number 21 | -----------+------------------ 22 | 10 | 1 23 | 12 | 300001 24 | 1000 | 450001 25 | 1000 | 451001 26 | 350 | 452001 27 | (5 rows) 28 | 29 | VACUUM FULL col_table_1; 30 | -- show that we properly update first_row_number after VACUUM FULL 31 | SELECT row_count, first_row_number FROM columnar.stripe a 32 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass) 33 | ORDER BY stripe_num; 34 | row_count | first_row_number 35 | -----------+------------------ 36 | 1000 | 1 37 | 1000 | 1001 38 | 372 | 2001 39 | (3 rows) 40 | 41 | TRUNCATE col_table_1; 42 | BEGIN; 43 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 16) i; 44 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 16) i; 45 | COMMIT; 46 | -- show that we start with first_row_number=1 after TRUNCATE 47 | SELECT row_count, first_row_number FROM columnar.stripe a 48 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass) 49 | ORDER BY stripe_num; 50 | row_count | first_row_number 51 | -----------+------------------ 52 | 32 | 1 53 | (1 row) 54 | 55 | SET client_min_messages TO ERROR; 56 | DROP SCHEMA columnar_first_row_number CASCADE; 57 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_join.out: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA am_columnar_join; 2 | SET search_path TO am_columnar_join; 3 | CREATE TABLE users (id int, name text) USING columnar; 4 | INSERT INTO users SELECT a, 'name' || a FROM generate_series(0,30-1) AS a; 5 | CREATE TABLE things (id int, user_id int, name text) USING columnar; 6 | INSERT INTO things SELECT a, a % 30, 'thing' || a FROM generate_series(1,300) AS a; 7 | -- force the nested loop to rescan the table 8 | SET enable_material TO off; 9 | SET enable_hashjoin TO off; 10 | SET enable_mergejoin TO off; 11 | SELECT count(*) 12 | FROM users 13 | JOIN things ON (users.id = things.user_id) 14 | WHERE things.id > 290; 15 | count 16 | ------- 17 | 10 18 | (1 row) 19 | 20 | -- verify the join uses a nested loop to trigger the rescan behaviour 21 | EXPLAIN (COSTS OFF) 22 | SELECT count(*) 23 | FROM users 24 | JOIN things ON (users.id = things.user_id) 25 | WHERE things.id > 299990; 26 | QUERY PLAN 27 | -------------------------------------------------------------------------------------- 28 | Aggregate 29 | -> Nested Loop 30 | -> Custom Scan (ColumnarScan) on users 31 | Columnar Projected Columns: id 32 | -> Custom Scan (ColumnarScan) on things 33 | Filter: ((id > 299990) AND (users.id = user_id)) 34 | Columnar Projected Columns: id, user_id 35 | Columnar Chunk Group Filters: ((id > 299990) AND (users.id = user_id)) 36 | (8 rows) 37 | 38 | EXPLAIN (COSTS OFF) 39 | SELECT u1.id, u2.id, COUNT(u2.*) 40 | FROM users u1 41 | JOIN users u2 ON (u1.id::text = u2.name) 42 | WHERE u2.id > 299990 43 | GROUP BY u1.id, u2.id; 44 | QUERY PLAN 45 | ---------------------------------------------------------------------------------------------- 46 | GroupAggregate 47 | Group Key: u1.id, u2.id 48 | -> Sort 49 | Sort Key: u1.id, u2.id 50 | -> Nested Loop 51 | -> Custom Scan (ColumnarScan) on users u1 52 | Columnar Projected Columns: id 53 | -> Custom Scan (ColumnarScan) on users u2 54 | Filter: ((id > 299990) AND ((u1.id)::text = name)) 55 | Columnar Projected Columns: id, name 56 | Columnar Chunk Group Filters: ((id > 299990) AND ((u1.id)::text = name)) 57 | (11 rows) 58 | 59 | SET client_min_messages TO warning; 60 | DROP SCHEMA am_columnar_join CASCADE; 61 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_lz4.out: -------------------------------------------------------------------------------- 1 | SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset 2 | \if :lz4_supported 3 | \else 4 | \q 5 | \endif 6 | CREATE SCHEMA am_alz4; 7 | SET search_path TO am_alz4; 8 | SET columnar.compression TO 'lz4'; 9 | CREATE TABLE test_lz4 (a int, b text, c int) USING columnar; 10 | INSERT INTO test_lz4 SELECT floor(i / 1000), floor(i / 10)::text, 4 FROM generate_series(1, 10000) i; 11 | SELECT count(*) FROM test_lz4; 12 | count 13 | ------- 14 | 10000 15 | (1 row) 16 | 17 | INSERT INTO test_lz4 SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 11000) i; 18 | SELECT count(*) FROM test_lz4; 19 | count 20 | ------- 21 | 20001 22 | (1 row) 23 | 24 | SELECT pg_relation_size('test_lz4') AS size_lz4 \gset 25 | SELECT DISTINCT * FROM test_lz4 ORDER BY a, b, c LIMIT 5; 26 | a | b | c 27 | ---+----+--- 28 | 0 | 0 | 4 29 | 0 | 1 | 4 30 | 0 | 10 | 4 31 | 0 | 11 | 4 32 | 0 | 12 | 4 33 | (5 rows) 34 | 35 | -- compare compression rate to pglz 36 | SET columnar.compression TO 'pglz'; 37 | CREATE TABLE test_pglz (LIKE test_lz4) USING columnar; 38 | INSERT INTO test_pglz SELECT * FROM test_lz4; 39 | SELECT pg_relation_size('test_pglz') AS size_pglz \gset 40 | -- verify that pglz & lz4 resulted in different compression ratios 41 | SELECT :size_pglz <> :size_lz4; 42 | ?column? 43 | ---------- 44 | t 45 | (1 row) 46 | 47 | -- Other operations 48 | VACUUM FULL test_lz4; 49 | ANALYZE test_lz4; 50 | SELECT count(DISTINCT test_lz4.*) FROM test_lz4; 51 | count 52 | ------- 53 | 6002 54 | (1 row) 55 | 56 | TRUNCATE test_lz4; 57 | SELECT count(DISTINCT test_lz4.*) FROM test_lz4; 58 | count 59 | ------- 60 | 0 61 | (1 row) 62 | 63 | SET client_min_messages TO WARNING; 64 | DROP SCHEMA am_alz4 CASCADE; 65 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_permissions.out: -------------------------------------------------------------------------------- 1 | select current_user \gset 2 | create user columnar_user; 3 | grant all on schema public to columnar_user; 4 | \c - columnar_user 5 | create table columnar_permissions(i int) using columnar; 6 | insert into columnar_permissions values(1); 7 | alter table columnar_permissions add column j int; 8 | insert into columnar_permissions values(2,20); 9 | vacuum columnar_permissions; 10 | truncate columnar_permissions; 11 | drop table columnar_permissions; 12 | \c - :current_user 13 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_upsert.out: -------------------------------------------------------------------------------- 1 | CREATE TABLE upsert_test ( 2 | i INT, 3 | t TEXT 4 | ) USING columnar; 5 | CREATE UNIQUE INDEX ON upsert_test (t); 6 | INSERT INTO upsert_test (i, t) VALUES (1, 'hello'); 7 | INSERT INTO upsert_test (i, t) VALUES (2, 'world'); 8 | -- should work, then roll back 9 | BEGIN; 10 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'foo'; 11 | SELECT * FROM upsert_test; 12 | i | t 13 | ---+------- 14 | 2 | world 15 | 1 | foo 16 | (2 rows) 17 | 18 | ROLLBACK; 19 | SELECT * FROM upsert_test; 20 | i | t 21 | ---+------- 22 | 1 | hello 23 | 2 | world 24 | (2 rows) 25 | 26 | -- should take affect immediately 27 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'bar'; 28 | SELECT * FROM upsert_test; 29 | i | t 30 | ---+------- 31 | 2 | world 32 | 1 | bar 33 | (2 rows) 34 | 35 | -- should work as expected 36 | BEGIN; 37 | INSERT INTO upsert_test (t) VALUES ( 'world' ) ON CONFLICT (t) DO UPDATE SET t = 'foo'; 38 | SELECT * FROM upsert_test; 39 | i | t 40 | ---+----- 41 | 1 | bar 42 | 2 | foo 43 | (2 rows) 44 | 45 | COMMIT; 46 | -- should also work as expected, a select can mask a bug 47 | DELETE FROM upsert_test; 48 | BEGIN; 49 | INSERT INTO upsert_test (i, t) VALUES (1, 'hello'); 50 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'bar'; 51 | COMMIT; 52 | SELECT * FROM upsert_test; 53 | i | t 54 | ---+----- 55 | 1 | bar 56 | (1 row) 57 | 58 | DROP TABLE upsert_test; 59 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_vectorization.out: -------------------------------------------------------------------------------- 1 | CREATE TABLE t (id int, ts timestamp) USING columnar; 2 | INSERT INTO t SELECT id, now() + id * '1 day'::interval FROM generate_series(1, 100000) id; 3 | EXPLAIN (costs off) SELECT * FROM t WHERE ts between '2026-01-01'::timestamp and '2026-02-01'::timestamp; 4 | QUERY PLAN 5 | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- 6 | Custom Scan (ColumnarScan) on t 7 | Columnar Projected Columns: id, ts 8 | Columnar Chunk Group Filters: ((ts >= 'Thu Jan 01 00:00:00 2026'::timestamp without time zone) AND (ts <= 'Sun Feb 01 00:00:00 2026'::timestamp without time zone)) 9 | Columnar Vectorized Filter: ((ts >= 'Thu Jan 01 00:00:00 2026'::timestamp without time zone) AND (ts <= 'Sun Feb 01 00:00:00 2026'::timestamp without time zone)) 10 | (4 rows) 11 | 12 | DROP TABLE t; 13 | CREATE TABLE t (id int, ts timestamptz) USING columnar; 14 | INSERT INTO t SELECT id, now() + id * '1 day'::interval FROM generate_series(1, 100000) id; 15 | EXPLAIN (costs off) SELECT * FROM t WHERE ts between '2026-01-01'::timestamptz and '2026-02-01'::timestamptz; 16 | QUERY PLAN 17 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 18 | Custom Scan (ColumnarScan) on t 19 | Columnar Projected Columns: id, ts 20 | Columnar Chunk Group Filters: ((ts >= 'Thu Jan 01 00:00:00 2026 PST'::timestamp with time zone) AND (ts <= 'Sun Feb 01 00:00:00 2026 PST'::timestamp with time zone)) 21 | Columnar Vectorized Filter: ((ts >= 'Thu Jan 01 00:00:00 2026 PST'::timestamp with time zone) AND (ts <= 'Sun Feb 01 00:00:00 2026 PST'::timestamp with time zone)) 22 | (4 rows) 23 | 24 | DROP TABLE t; 25 | -------------------------------------------------------------------------------- /columnar/src/test/regress/expected/columnar_zstd.out: -------------------------------------------------------------------------------- 1 | SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset 2 | \if :zstd_supported 3 | \else 4 | \q 5 | \endif 6 | CREATE SCHEMA am_zstd; 7 | SET search_path TO am_zstd; 8 | SET columnar.compression TO 'zstd'; 9 | CREATE TABLE test_zstd (a int, b text, c int) USING columnar; 10 | INSERT INTO test_zstd SELECT i % 1000, (i % 10)::text, 4 FROM generate_series(1, 10000) i; 11 | SELECT count(*) FROM test_zstd; 12 | count 13 | ------- 14 | 10000 15 | (1 row) 16 | 17 | INSERT INTO test_zstd SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 11000) i; 18 | SELECT count(*) FROM test_zstd; 19 | count 20 | ------- 21 | 20001 22 | (1 row) 23 | 24 | CREATE TABLE test_none (LIKE test_zstd) USING columnar; 25 | INSERT INTO test_none SELECT * FROM test_zstd; 26 | SELECT DISTINCT * FROM test_zstd ORDER BY a, b, c LIMIT 5; 27 | a | b | c 28 | ---+---+--- 29 | 0 | 0 | 4 30 | 1 | 1 | 4 31 | 2 | 2 | 4 32 | 3 | 3 | 4 33 | 4 | 4 | 4 34 | (5 rows) 35 | 36 | VACUUM FULL test_zstd; 37 | -- Hydra: we need to get `data_length` from stripe metadata information to compare compression 38 | -- rather than calling pg_relation_size function 39 | SELECT data_length AS size_comp_level_default FROM columnar.stripe WHERE storage_id = ( 40 | SELECT storage_id from columnar_test_helpers.columnar_storage_info('test_zstd')) \gset 41 | -- change compression level 42 | SELECT columnar.alter_columnar_table_set('test_zstd', compression_level => 19); 43 | alter_columnar_table_set 44 | -------------------------- 45 | 46 | (1 row) 47 | 48 | VACUUM FULL test_zstd; 49 | SELECT data_length AS size_comp_level_19 FROM columnar.stripe WHERE storage_id = ( 50 | SELECT storage_id from columnar_test_helpers.columnar_storage_info('test_zstd')) \gset 51 | -- verify that higher compression level compressed better 52 | SELECT :size_comp_level_default > :size_comp_level_19 AS size_changed; 53 | size_changed 54 | -------------- 55 | t 56 | (1 row) 57 | 58 | -- compare compression rate to pglz 59 | SET columnar.compression TO 'pglz'; 60 | CREATE TABLE test_pglz (LIKE test_zstd) USING columnar; 61 | INSERT INTO test_pglz SELECT * FROM test_zstd; 62 | SELECT pg_relation_size('test_pglz') AS size_pglz \gset 63 | -- verify that zstd compressed better than pglz 64 | SELECT :size_pglz > :size_comp_level_default; 65 | ?column? 66 | ---------- 67 | t 68 | (1 row) 69 | 70 | -- Other operations 71 | ANALYZE test_zstd; 72 | SELECT count(DISTINCT test_zstd.*) FROM test_zstd; 73 | count 74 | ------- 75 | 6001 76 | (1 row) 77 | 78 | TRUNCATE test_zstd; 79 | SELECT count(DISTINCT test_zstd.*) FROM test_zstd; 80 | count 81 | ------- 82 | 0 83 | (1 row) 84 | 85 | SET client_min_messages TO WARNING; 86 | DROP SCHEMA am_zstd CASCADE; 87 | -------------------------------------------------------------------------------- /columnar/src/test/regress/input/columnar_copyto.source: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test copying data from columnar tables. 3 | -- 4 | CREATE TABLE test_contestant(handle TEXT, birthdate DATE, rating INT, 5 | percentile FLOAT, country CHAR(3), achievements TEXT[]) 6 | USING columnar; 7 | 8 | -- load table data from file 9 | COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; 10 | 11 | -- export using COPY table TO ... 12 | COPY test_contestant TO STDOUT; 13 | 14 | -- export using COPY (SELECT * FROM table) TO ... 15 | COPY (select * from test_contestant) TO STDOUT; 16 | 17 | DROP TABLE test_contestant CASCADE; 18 | -------------------------------------------------------------------------------- /columnar/src/test/regress/input/columnar_data_types.source: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test loading and reading different data types to/from columnar foreign tables. 3 | -- 4 | 5 | 6 | -- Settings to make the result deterministic 7 | SET datestyle = "ISO, YMD"; 8 | SET timezone to 'GMT'; 9 | SET intervalstyle TO 'POSTGRES_VERBOSE'; 10 | 11 | 12 | -- Test array types 13 | CREATE TABLE test_array_types (int_array int[], bigint_array bigint[], 14 | text_array text[]) USING columnar; 15 | 16 | COPY test_array_types FROM '@abs_srcdir@/data/array_types.csv' WITH CSV; 17 | 18 | SELECT * FROM test_array_types; 19 | 20 | 21 | -- Test date/time types 22 | CREATE TABLE test_datetime_types (timestamp timestamp, 23 | timestamp_with_timezone timestamp with time zone, date date, time time, 24 | interval interval) USING columnar; 25 | 26 | COPY test_datetime_types FROM '@abs_srcdir@/data/datetime_types.csv' WITH CSV; 27 | 28 | SELECT * FROM test_datetime_types; 29 | 30 | 31 | -- Test enum and composite types 32 | CREATE TYPE enum_type AS ENUM ('a', 'b', 'c'); 33 | CREATE TYPE composite_type AS (a int, b text); 34 | 35 | CREATE TABLE test_enum_and_composite_types (enum enum_type, 36 | composite composite_type) USING columnar; 37 | 38 | COPY test_enum_and_composite_types FROM 39 | '@abs_srcdir@/data/enum_and_composite_types.csv' WITH CSV; 40 | 41 | SELECT * FROM test_enum_and_composite_types; 42 | 43 | 44 | -- Test range types 45 | CREATE TABLE test_range_types (int4range int4range, int8range int8range, 46 | numrange numrange, tsrange tsrange) USING columnar; 47 | 48 | COPY test_range_types FROM '@abs_srcdir@/data/range_types.csv' WITH CSV; 49 | 50 | SELECT * FROM test_range_types; 51 | 52 | 53 | -- Test other types 54 | CREATE TABLE test_other_types (bool boolean, bytea bytea, money money, 55 | inet inet, bitstring bit varying(5), uuid uuid, json json) USING columnar; 56 | 57 | COPY test_other_types FROM '@abs_srcdir@/data/other_types.csv' WITH CSV; 58 | 59 | SELECT * FROM test_other_types; 60 | 61 | 62 | -- Test null values 63 | CREATE TABLE test_null_values (a int, b int[], c composite_type) 64 | USING columnar; 65 | 66 | COPY test_null_values FROM '@abs_srcdir@/data/null_values.csv' WITH CSV; 67 | 68 | SELECT * FROM test_null_values; 69 | 70 | CREATE TABLE test_json(j json) USING columnar; 71 | INSERT INTO test_json SELECT ('{"att": ' || g::text || '}')::json from generate_series(1,1000000) g; 72 | SELECT * FROM test_json WHERE (j->'att')::text::int8 > 999990; 73 | DROP TABLE test_json; 74 | -------------------------------------------------------------------------------- /columnar/src/test/regress/input/columnar_load.source: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test loading data into columnar tables. 3 | -- 4 | 5 | -- COPY with incorrect delimiter 6 | COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' 7 | WITH DELIMITER '|'; -- ERROR 8 | 9 | -- COPY with invalid program 10 | COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR 11 | 12 | -- COPY into uncompressed table from file 13 | COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; 14 | 15 | -- COPY into uncompressed table from program 16 | COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; 17 | 18 | select 19 | version_major, version_minor, reserved_stripe_id, reserved_row_number 20 | from columnar_test_helpers.columnar_storage_info('contestant'); 21 | 22 | -- COPY into compressed table 23 | COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; 24 | 25 | -- COPY into compressed table from program 26 | COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' 27 | WITH CSV; 28 | 29 | select 30 | version_major, version_minor, reserved_stripe_id, reserved_row_number 31 | from columnar_test_helpers.columnar_storage_info('contestant_compressed'); 32 | 33 | -- Test column list 34 | CREATE TABLE famous_constants (id int, name text, value real) 35 | USING columnar; 36 | COPY famous_constants (value, name, id) FROM STDIN WITH CSV; 37 | 3.141,pi,1 38 | 2.718,e,2 39 | 0.577,gamma,3 40 | 5.291e-11,bohr radius,4 41 | \. 42 | 43 | COPY famous_constants (name, value) FROM STDIN WITH CSV; 44 | avagadro,6.022e23 45 | electron mass,9.109e-31 46 | proton mass,1.672e-27 47 | speed of light,2.997e8 48 | \. 49 | 50 | SELECT * FROM famous_constants ORDER BY id, name; 51 | 52 | SELECT * FROM columnar_test_helpers.chunk_group_consistency; 53 | 54 | DROP TABLE famous_constants; 55 | -------------------------------------------------------------------------------- /columnar/src/test/regress/output/columnar_copyto.source: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test copying data from columnar tables. 3 | -- 4 | CREATE TABLE test_contestant(handle TEXT, birthdate DATE, rating INT, 5 | percentile FLOAT, country CHAR(3), achievements TEXT[]) 6 | USING columnar; 7 | -- load table data from file 8 | COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; 9 | -- export using COPY table TO ... 10 | COPY test_contestant TO STDOUT; 11 | a 01-10-1990 2090 97.1 XA {a} 12 | b 11-01-1990 2203 98.1 XA {a,b} 13 | c 11-01-1988 2907 99.4 XB {w,y} 14 | d 05-05-1985 2314 98.3 XB {} 15 | e 05-05-1995 2236 98.2 XC {a} 16 | -- export using COPY (SELECT * FROM table) TO ... 17 | COPY (select * from test_contestant) TO STDOUT; 18 | a 01-10-1990 2090 97.1 XA {a} 19 | b 11-01-1990 2203 98.1 XA {a,b} 20 | c 11-01-1988 2907 99.4 XB {w,y} 21 | d 05-05-1985 2314 98.3 XB {} 22 | e 05-05-1995 2236 98.2 XC {a} 23 | DROP TABLE test_contestant CASCADE; 24 | -------------------------------------------------------------------------------- /columnar/src/test/regress/output/columnar_load.source: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test loading data into columnar tables. 3 | -- 4 | -- COPY with incorrect delimiter 5 | COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' 6 | WITH DELIMITER '|'; -- ERROR 7 | ERROR: missing data for column "birthdate" 8 | CONTEXT: COPY contestant, line 1: "a,1990-01-10,2090,97.1,XA ,{a}" 9 | -- COPY with invalid program 10 | COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR 11 | ERROR: program "invalid_program" failed 12 | DETAIL: command not found 13 | -- COPY into uncompressed table from file 14 | COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; 15 | -- COPY into uncompressed table from program 16 | COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV; 17 | select 18 | version_major, version_minor, reserved_stripe_id, reserved_row_number 19 | from columnar_test_helpers.columnar_storage_info('contestant'); 20 | version_major | version_minor | reserved_stripe_id | reserved_row_number 21 | ---------------+---------------+--------------------+--------------------- 22 | 2 | 0 | 3 | 300001 23 | (1 row) 24 | 25 | -- COPY into compressed table 26 | COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV; 27 | -- COPY into compressed table from program 28 | COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' 29 | WITH CSV; 30 | select 31 | version_major, version_minor, reserved_stripe_id, reserved_row_number 32 | from columnar_test_helpers.columnar_storage_info('contestant_compressed'); 33 | version_major | version_minor | reserved_stripe_id | reserved_row_number 34 | ---------------+---------------+--------------------+--------------------- 35 | 2 | 0 | 3 | 300001 36 | (1 row) 37 | 38 | -- Test column list 39 | CREATE TABLE famous_constants (id int, name text, value real) 40 | USING columnar; 41 | COPY famous_constants (value, name, id) FROM STDIN WITH CSV; 42 | COPY famous_constants (name, value) FROM STDIN WITH CSV; 43 | SELECT * FROM famous_constants ORDER BY id, name; 44 | id | name | value 45 | ----+----------------+----------- 46 | 1 | pi | 3.141 47 | 2 | e | 2.718 48 | 3 | gamma | 0.577 49 | 4 | bohr radius | 5.291e-11 50 | | avagadro | 6.022e+23 51 | | electron mass | 9.109e-31 52 | | proton mass | 1.672e-27 53 | | speed of light | 2.997e+08 54 | (8 rows) 55 | 56 | SELECT * FROM columnar_test_helpers.chunk_group_consistency; 57 | consistent 58 | ------------ 59 | t 60 | (1 row) 61 | 62 | DROP TABLE famous_constants; 63 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/.gitignore: -------------------------------------------------------------------------------- 1 | /columnar_copyto.sql 2 | /columnar_data_types.sql 3 | /columnar_load.sql 4 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_alter_set_type.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Testing ALTER TABLE on columnar tables. 3 | -- 4 | 5 | CREATE TABLE test_alter_table (a int, b int, c text) USING columnar; 6 | 7 | WITH sample_data AS (VALUES 8 | (1, 2, '3'), 9 | (4, 5, '6') 10 | ) 11 | INSERT INTO test_alter_table SELECT * FROM sample_data; 12 | 13 | WITH sample_data AS (VALUES 14 | (5, 9, '11'), 15 | (12, 83, '93') 16 | ) 17 | INSERT INTO test_alter_table SELECT * FROM sample_data; 18 | 19 | ALTER TABLE test_alter_table ALTER COLUMN a TYPE jsonb USING row_to_json(row(a)); 20 | SELECT * FROM test_alter_table ORDER BY a; 21 | 22 | ALTER TABLE test_alter_table ALTER COLUMN c TYPE int USING c::integer; 23 | SELECT sum(c) FROM test_alter_table; 24 | 25 | ALTER TABLE test_alter_table ALTER COLUMN b TYPE bigint; 26 | SELECT * FROM test_alter_table ORDER BY a; 27 | 28 | ALTER TABLE test_alter_table ALTER COLUMN b TYPE float USING (b::float + 0.5); 29 | SELECT * FROM test_alter_table ORDER BY a; 30 | 31 | DROP TABLE test_alter_table; 32 | 33 | -- Make sure that the correct table options are used when rewriting the table. 34 | -- This is reflected by the VACUUM VERBOSE output right after a rewrite showing 35 | -- that all chunks are compressed with the configured compression algorithm 36 | CREATE TABLE test(i int) USING columnar; 37 | SELECT columnar.alter_columnar_table_set('test', compression => 'lz4'); 38 | INSERT INTO test VALUES(1); 39 | VACUUM VERBOSE test; 40 | 41 | ALTER TABLE test ALTER COLUMN i TYPE int8; 42 | VACUUM VERBOSE test; 43 | 44 | DROP TABLE test; 45 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_analyze.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test the ANALYZE command for columnar tables. 3 | -- 4 | 5 | -- ANALYZE uncompressed table 6 | ANALYZE contestant; 7 | SELECT count(*) FROM pg_stats WHERE tablename='contestant'; 8 | 9 | -- ANALYZE compressed table 10 | ANALYZE contestant_compressed; 11 | SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed'; 12 | 13 | -- ANALYZE a table with lots of data to trigget qsort in analyze.c 14 | CREATE TABLE test_analyze(a int, b text, c char) USING columnar; 15 | INSERT INTO test_analyze SELECT floor(i / 1000), floor(i / 10)::text, 4 FROM generate_series(1, 100000) i; 16 | INSERT INTO test_analyze SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 110000) i; 17 | 18 | ANALYZE test_analyze; 19 | DROP TABLE test_analyze; 20 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_cache.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE big_table ( 2 | id INT, 3 | firstname TEXT, 4 | lastname TEXT 5 | ) USING columnar; 6 | 7 | INSERT INTO big_table (id, firstname, lastname) 8 | SELECT i, 9 | CONCAT('firstname-', i), 10 | CONCAT('lastname-', i) 11 | FROM generate_series(1, 1000000) as i; 12 | 13 | -- get some baselines from multiple chunks 14 | SELECT firstname, 15 | lastname, 16 | SUM(id) 17 | FROM big_table 18 | WHERE id < 1000 19 | GROUP BY firstname, 20 | lastname 21 | UNION 22 | SELECT firstname, 23 | lastname, 24 | SUM(id) 25 | FROM big_table 26 | WHERE id BETWEEN 15000 AND 16000 27 | GROUP BY firstname, 28 | lastname 29 | ORDER BY firstname; 30 | 31 | 32 | -- enable caching 33 | SET columnar.enable_column_cache = 't'; 34 | 35 | -- the results should be the same as above 36 | SELECT firstname, 37 | lastname, 38 | SUM(id) 39 | FROM big_table 40 | WHERE id < 1000 41 | GROUP BY firstname, 42 | lastname 43 | UNION 44 | SELECT firstname, 45 | lastname, 46 | SUM(id) 47 | FROM big_table 48 | WHERE id BETWEEN 15000 AND 16000 49 | GROUP BY firstname, 50 | lastname 51 | ORDER BY firstname; 52 | 53 | -- disable caching 54 | SET columnar.enable_column_cache = 'f'; 55 | 56 | CREATE TABLE test_2 ( 57 | value INT, 58 | updated_value INT 59 | ) USING columnar; 60 | 61 | INSERT INTO test_2 (value) 62 | SELECT generate_series(1, 1000000, 1); 63 | 64 | BEGIN; 65 | SELECT SUM(value) 66 | FROM test_2; 67 | 68 | UPDATE test_2 69 | SET updated_value = value * 2; 70 | 71 | SELECT SUM(updated_value) 72 | FROM test_2; 73 | 74 | DELETE FROM test_2 75 | WHERE value % 2 = 0; 76 | 77 | SELECT SUM(value) 78 | FROM test_2; 79 | COMMIT; 80 | 81 | DROP TABLE test_2; 82 | 83 | set columnar.enable_column_cache = 't'; 84 | 85 | CREATE TABLE test_2 ( 86 | value INT, 87 | updated_value INT 88 | ) USING columnar; 89 | 90 | INSERT INTO test_2 (value) 91 | SELECT generate_series(1, 1000000, 1); 92 | 93 | BEGIN; 94 | SELECT SUM(value) 95 | FROM test_2; 96 | 97 | UPDATE test_2 98 | SET updated_value = value * 2; 99 | 100 | SELECT SUM(updated_value) 101 | FROM test_2; 102 | 103 | DELETE FROM test_2 104 | WHERE value % 2 = 0; 105 | 106 | SELECT SUM(value) 107 | FROM test_2; 108 | COMMIT; 109 | 110 | DROP TABLE test_2; 111 | 112 | CREATE TABLE t1 (i int) USING columnar; 113 | 114 | INSERT INTO t1 SELECT generate_series(1, 1000000, 1); 115 | EXPLAIN SELECT COUNT(*) FROM t1; 116 | DROP TABLE t1; 117 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_clean.sql: -------------------------------------------------------------------------------- 1 | 2 | DROP TABLE test_null_values; 3 | DROP TABLE test_other_types; 4 | DROP TABLE test_range_types; 5 | DROP TABLE test_enum_and_composite_types; 6 | DROP TYPE composite_type; 7 | DROP TYPE enum_type; 8 | DROP TABLE test_datetime_types; 9 | DROP TABLE test_array_types; 10 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_cursor.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Testing cursors on columnar tables. 3 | -- 4 | 5 | CREATE TABLE test_cursor (a int, b int) USING columnar; 6 | INSERT INTO test_cursor SELECT i, j FROM generate_series(0, 100)i, generate_series(100, 200)j; 7 | 8 | -- A case where the WHERE clause might filter out some chunks 9 | EXPLAIN (analyze on, costs off, timing off, summary off) SELECT * FROM test_cursor WHERE a = 25; 10 | BEGIN; 11 | DECLARE a_25 SCROLL CURSOR 12 | FOR SELECT * FROM test_cursor WHERE a = 25 ORDER BY 2; 13 | FETCH 3 FROM a_25; 14 | FETCH PRIOR FROM a_25; 15 | FETCH NEXT FROM a_25; 16 | FETCH NEXT FROM a_25; 17 | FETCH RELATIVE -2 FROM a_25; 18 | FETCH LAST FROM a_25; 19 | FETCH RELATIVE -25 FROM a_25; 20 | MOVE a_25; 21 | FETCH a_25; 22 | MOVE LAST FROM a_25; 23 | FETCH a_25; 24 | MOVE RELATIVE -3 FROM a_25; 25 | FETCH a_25; 26 | MOVE RELATIVE -3 FROM a_25; 27 | FETCH a_25; 28 | MOVE FORWARD 2 FROM a_25; 29 | FETCH a_25; 30 | MOVE RELATIVE -3 FROM a_25; 31 | FETCH a_25; 32 | UPDATE test_cursor SET a = 8000 WHERE CURRENT OF a_25; 33 | COMMIT; 34 | 35 | -- A case where the WHERE clause doesn't filter out any chunks 36 | EXPLAIN (analyze on, costs off, timing off, summary off) SELECT * FROM test_cursor WHERE a > 25; 37 | BEGIN; 38 | DECLARE a_25 SCROLL CURSOR 39 | FOR SELECT * FROM test_cursor WHERE a > 25 ORDER BY 1, 2; 40 | FETCH 3 FROM a_25; 41 | FETCH PRIOR FROM a_25; 42 | FETCH NEXT FROM a_25; 43 | FETCH NEXT FROM a_25; 44 | FETCH RELATIVE -2 FROM a_25; 45 | FETCH LAST FROM a_25; 46 | FETCH RELATIVE -25 FROM a_25; 47 | MOVE a_25; 48 | FETCH a_25; 49 | MOVE LAST FROM a_25; 50 | FETCH a_25; 51 | MOVE RELATIVE -3 FROM a_25; 52 | FETCH a_25; 53 | MOVE RELATIVE -3 FROM a_25; 54 | FETCH a_25; 55 | MOVE FORWARD 2 FROM a_25; 56 | FETCH a_25; 57 | MOVE RELATIVE -3 FROM a_25; 58 | FETCH a_25; 59 | UPDATE test_cursor SET a = 8000 WHERE CURRENT OF a_25; 60 | COMMIT; 61 | 62 | DROP TABLE test_cursor CASCADE; 63 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_customindex.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test custom index: Test shows only that we are exchanging Index scan with CustomIndexScan while 3 | -- all other information is still the same. The feature is not visible through EXPLAIN because output 4 | -- will be same in both cases, but difference is noticeable how the columnar storage engine is read. 5 | -- With index scan ALL columns are requested from storage (this can be significant overhead in performance) 6 | -- while with CustomIndexScan we will request only columns that are needed - that are going to be used as output. 7 | -- Test also shows that we are not using CustomIndexScan with heap tables and only on columnar tables. 8 | -- 9 | 10 | SET columnar.enable_columnar_index_scan TO TRUE; 11 | 12 | CREATE TABLE t(a INT PRIMARY KEY, b INT, c TEXT) USING columnar; 13 | CREATE INDEX t_idx ON t USING btree(b); 14 | 15 | CREATE TABLE t_heap(a INT PRIMARY KEY, b INT, c TEXT); 16 | CREATE INDEX t_idx_heap ON t_heap USING btree(b); 17 | 18 | INSERT INTO t SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g; 19 | INSERT INTO t_heap SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g; 20 | 21 | -- make sure that we test index scan 22 | set columnar.enable_custom_scan TO 'off'; 23 | set enable_seqscan TO off; 24 | set seq_page_cost TO 10000000; 25 | 26 | EXPLAIN (VERBOSE) SELECT a FROM t WHERE b > 18 ORDER BY b LIMIT 10; 27 | 28 | EXPLAIN (VERBOSE) SELECT a FROM t_heap WHERE b > 18 ORDER BY b LIMIT 10; 29 | 30 | DROP TABLE t; 31 | DROP TABLE t_heap; 32 | 33 | SET columnar.enable_custom_index_scan TO default; 34 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_drop.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Tests the different DROP commands for columnar tables. 3 | -- 4 | -- DROP TABL 5 | -- DROP SCHEMA 6 | -- DROP EXTENSION 7 | -- DROP DATABASE 8 | -- 9 | 10 | -- Note that travis does not create 11 | -- citus extension in default database (postgres). This has caused 12 | -- different behavior between travis tests and local tests. Thus 13 | -- 'postgres' directory is excluded from comparison to have the same result. 14 | 15 | -- store postgres database oid 16 | SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset 17 | 18 | SELECT count(distinct storage_id) AS columnar_stripes_before_drop FROM columnar.stripe \gset 19 | 20 | -- DROP columnar tables 21 | DROP TABLE contestant; 22 | DROP TABLE contestant_compressed; 23 | 24 | -- make sure DROP deletes metadata 25 | SELECT :columnar_stripes_before_drop - count(distinct storage_id) FROM columnar.stripe; 26 | 27 | -- Create a columnar table under a schema and drop it. 28 | CREATE SCHEMA test_schema; 29 | CREATE TABLE test_schema.test_table(data int) USING columnar; 30 | INSERT INTO test_schema.test_table VALUES (1); 31 | 32 | SELECT count(*) AS columnar_stripes_before_drop FROM columnar.stripe \gset 33 | DROP SCHEMA test_schema CASCADE; 34 | SELECT :columnar_stripes_before_drop - count(distinct storage_id) FROM columnar.stripe; 35 | 36 | SELECT current_database() datname \gset 37 | 38 | CREATE DATABASE db_to_drop; 39 | \c db_to_drop 40 | CREATE EXTENSION columnar; 41 | SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset 42 | 43 | CREATE TABLE test_table(data int) USING columnar; 44 | 45 | DROP EXTENSION columnar CASCADE; 46 | 47 | -- test database drop 48 | CREATE EXTENSION columnar; 49 | SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset 50 | 51 | CREATE TABLE test_table(data int) USING columnar; 52 | 53 | \c :datname 54 | 55 | DROP DATABASE db_to_drop; 56 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_empty.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Test different operations on empty columnar tables. 3 | -- 4 | 5 | SET columnar.compression to 'none'; 6 | create table t_uncompressed(a int) using columnar; 7 | create table t_compressed(a int) using columnar; 8 | 9 | -- set options 10 | SELECT columnar.alter_columnar_table_set('t_compressed', compression => 'pglz'); 11 | SELECT columnar.alter_columnar_table_set('t_compressed', stripe_row_limit => 2000); 12 | SELECT columnar.alter_columnar_table_set('t_compressed', chunk_group_row_limit => 1000); 13 | 14 | SELECT * FROM columnar.options WHERE regclass = 't_compressed'::regclass; 15 | 16 | -- select 17 | select * from t_uncompressed; 18 | select count(*) from t_uncompressed; 19 | select * from t_compressed; 20 | select count(*) from t_compressed; 21 | 22 | -- check storage 23 | select 24 | version_major, version_minor, reserved_stripe_id, reserved_row_number 25 | from columnar_test_helpers.columnar_storage_info('t_compressed'); 26 | select 27 | version_major, version_minor, reserved_stripe_id, reserved_row_number 28 | from columnar_test_helpers.columnar_storage_info('t_uncompressed'); 29 | 30 | -- explain 31 | explain (costs off, summary off, timing off) select * from t_uncompressed; 32 | explain (costs off, summary off, timing off) select * from t_compressed; 33 | 34 | -- vacuum 35 | vacuum t_compressed; 36 | vacuum t_uncompressed; 37 | 38 | -- vacuum full 39 | vacuum full t_compressed; 40 | vacuum full t_uncompressed; 41 | 42 | -- check storage 43 | select 44 | version_major, version_minor, reserved_stripe_id, reserved_row_number 45 | from columnar_test_helpers.columnar_storage_info('t_compressed'); 46 | select 47 | version_major, version_minor, reserved_stripe_id, reserved_row_number 48 | from columnar_test_helpers.columnar_storage_info('t_uncompressed'); 49 | 50 | -- analyze 51 | analyze t_uncompressed; 52 | analyze t_compressed; 53 | 54 | -- truncate 55 | truncate t_uncompressed; 56 | truncate t_compressed; 57 | 58 | -- alter type 59 | alter table t_uncompressed alter column a type text; 60 | alter table t_compressed alter column a type text; 61 | 62 | -- check storage 63 | select 64 | version_major, version_minor, reserved_stripe_id, reserved_row_number 65 | from columnar_test_helpers.columnar_storage_info('t_compressed'); 66 | select 67 | version_major, version_minor, reserved_stripe_id, reserved_row_number 68 | from columnar_test_helpers.columnar_storage_info('t_uncompressed'); 69 | 70 | -- verify cost of scanning an empty table is zero, not NaN 71 | explain table t_uncompressed; 72 | explain table t_compressed; 73 | 74 | -- drop 75 | drop table t_compressed; 76 | drop table t_uncompressed; 77 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_fallback_scan.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- columnar_fallback_scan.sql 3 | -- 4 | -- Test columnar.enable_custom_scan = false, which will use an 5 | -- ordinary sequential scan. It won't benefit from projection pushdown 6 | -- or qual pushdown, but it should return the correct results. 7 | -- 8 | 9 | set columnar.enable_custom_scan = false; 10 | 11 | create table fallback_scan(i int) using columnar; 12 | -- large enough to test parallel_workers > 1 13 | select columnar.alter_columnar_table_set('fallback_scan', compression => 'none'); 14 | insert into fallback_scan select generate_series(1,150000); 15 | vacuum analyze fallback_scan; 16 | 17 | select count(*), min(i), max(i), avg(i) from fallback_scan; 18 | 19 | -- 20 | -- Negative test: try to force a parallel plan with at least two 21 | -- workers, but columnar should reject it and use a non-parallel scan. 22 | -- 23 | set min_parallel_table_scan_size = 1; 24 | set parallel_tuple_cost = 0; 25 | set max_parallel_workers = 4; 26 | set max_parallel_workers_per_gather = 4; 27 | explain (costs off) select count(*), min(i), max(i), avg(i) from fallback_scan; 28 | select count(*), min(i), max(i), avg(i) from fallback_scan; 29 | 30 | set min_parallel_table_scan_size to default; 31 | set parallel_tuple_cost to default; 32 | set max_parallel_workers to default; 33 | set max_parallel_workers_per_gather to default; 34 | 35 | set columnar.enable_custom_scan to default; 36 | 37 | drop table fallback_scan; 38 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_first_row_number.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA columnar_first_row_number; 2 | SET search_path tO columnar_first_row_number; 3 | 4 | CREATE TABLE col_table_1 (a int) USING columnar; 5 | 6 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 10) i; 7 | 8 | BEGIN; 9 | -- we don't use same first_row_number even if the xact is rollback'ed 10 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 11) i; 11 | ROLLBACK; 12 | 13 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 12) i; 14 | 15 | SELECT columnar.alter_columnar_table_set('col_table_1', stripe_row_limit => 1000); 16 | 17 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 2350) i; 18 | 19 | SELECT row_count, first_row_number FROM columnar.stripe a 20 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass) 21 | ORDER BY stripe_num; 22 | 23 | VACUUM FULL col_table_1; 24 | 25 | -- show that we properly update first_row_number after VACUUM FULL 26 | SELECT row_count, first_row_number FROM columnar.stripe a 27 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass) 28 | ORDER BY stripe_num; 29 | 30 | TRUNCATE col_table_1; 31 | 32 | BEGIN; 33 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 16) i; 34 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 16) i; 35 | COMMIT; 36 | 37 | -- show that we start with first_row_number=1 after TRUNCATE 38 | SELECT row_count, first_row_number FROM columnar.stripe a 39 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass) 40 | ORDER BY stripe_num; 41 | 42 | SET client_min_messages TO ERROR; 43 | DROP SCHEMA columnar_first_row_number CASCADE; 44 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_join.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA am_columnar_join; 2 | SET search_path TO am_columnar_join; 3 | 4 | CREATE TABLE users (id int, name text) USING columnar; 5 | INSERT INTO users SELECT a, 'name' || a FROM generate_series(0,30-1) AS a; 6 | 7 | CREATE TABLE things (id int, user_id int, name text) USING columnar; 8 | INSERT INTO things SELECT a, a % 30, 'thing' || a FROM generate_series(1,300) AS a; 9 | 10 | -- force the nested loop to rescan the table 11 | SET enable_material TO off; 12 | SET enable_hashjoin TO off; 13 | SET enable_mergejoin TO off; 14 | 15 | SELECT count(*) 16 | FROM users 17 | JOIN things ON (users.id = things.user_id) 18 | WHERE things.id > 290; 19 | 20 | -- verify the join uses a nested loop to trigger the rescan behaviour 21 | EXPLAIN (COSTS OFF) 22 | SELECT count(*) 23 | FROM users 24 | JOIN things ON (users.id = things.user_id) 25 | WHERE things.id > 299990; 26 | 27 | EXPLAIN (COSTS OFF) 28 | SELECT u1.id, u2.id, COUNT(u2.*) 29 | FROM users u1 30 | JOIN users u2 ON (u1.id::text = u2.name) 31 | WHERE u2.id > 299990 32 | GROUP BY u1.id, u2.id; 33 | 34 | SET client_min_messages TO warning; 35 | DROP SCHEMA am_columnar_join CASCADE; 36 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_lz4.sql: -------------------------------------------------------------------------------- 1 | SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset 2 | \if :lz4_supported 3 | \else 4 | \q 5 | \endif 6 | 7 | CREATE SCHEMA am_alz4; 8 | SET search_path TO am_alz4; 9 | 10 | SET columnar.compression TO 'lz4'; 11 | CREATE TABLE test_lz4 (a int, b text, c int) USING columnar; 12 | 13 | INSERT INTO test_lz4 SELECT floor(i / 1000), floor(i / 10)::text, 4 FROM generate_series(1, 10000) i; 14 | SELECT count(*) FROM test_lz4; 15 | 16 | INSERT INTO test_lz4 SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 11000) i; 17 | SELECT count(*) FROM test_lz4; 18 | 19 | SELECT pg_relation_size('test_lz4') AS size_lz4 \gset 20 | 21 | SELECT DISTINCT * FROM test_lz4 ORDER BY a, b, c LIMIT 5; 22 | 23 | -- compare compression rate to pglz 24 | SET columnar.compression TO 'pglz'; 25 | CREATE TABLE test_pglz (LIKE test_lz4) USING columnar; 26 | INSERT INTO test_pglz SELECT * FROM test_lz4; 27 | 28 | SELECT pg_relation_size('test_pglz') AS size_pglz \gset 29 | 30 | -- verify that pglz & lz4 resulted in different compression ratios 31 | SELECT :size_pglz <> :size_lz4; 32 | 33 | -- Other operations 34 | VACUUM FULL test_lz4; 35 | ANALYZE test_lz4; 36 | 37 | SELECT count(DISTINCT test_lz4.*) FROM test_lz4; 38 | 39 | TRUNCATE test_lz4; 40 | 41 | SELECT count(DISTINCT test_lz4.*) FROM test_lz4; 42 | 43 | SET client_min_messages TO WARNING; 44 | DROP SCHEMA am_alz4 CASCADE; 45 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_matview.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Testing we materialized views properly 3 | -- 4 | 5 | SET columnar.compression TO 'none'; 6 | 7 | CREATE TABLE t(a int, b int) USING columnar; 8 | 9 | INSERT INTO t SELECT floor(i / 4), 2 * i FROM generate_series(1, 10) i; 10 | 11 | CREATE MATERIALIZED VIEW t_view(a, bsum, cnt) USING columnar AS 12 | SELECT a, sum(b), count(*) FROM t GROUP BY a; 13 | 14 | SELECT * FROM t_view a ORDER BY a; 15 | 16 | INSERT INTO t SELECT floor(i / 4), 2 * i FROM generate_series(11, 20) i; 17 | 18 | SELECT * FROM t_view a ORDER BY a; 19 | 20 | -- show columnar options for materialized view 21 | SELECT * FROM columnar.options 22 | WHERE regclass = 't_view'::regclass; 23 | 24 | -- show we can set options on a materialized view 25 | SELECT columnar.alter_columnar_table_set('t_view', compression => 'pglz'); 26 | SELECT * FROM columnar.options 27 | WHERE regclass = 't_view'::regclass; 28 | 29 | REFRESH MATERIALIZED VIEW t_view; 30 | 31 | -- verify options have not been changed 32 | SELECT * FROM columnar.options 33 | WHERE regclass = 't_view'::regclass; 34 | 35 | SELECT * FROM t_view a ORDER BY a; 36 | 37 | -- verify that we have created metadata entries for the materialized view 38 | SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS storageid 39 | FROM pg_class WHERE relname='t_view' \gset 40 | 41 | SELECT count(*) FROM columnar.stripe WHERE storage_id=:storageid; 42 | SELECT count(*) FROM columnar.chunk WHERE storage_id=:storageid; 43 | 44 | DROP TABLE t CASCADE; 45 | 46 | -- dropping must remove metadata 47 | SELECT count(*) FROM columnar.stripe WHERE storage_id=:storageid; 48 | SELECT count(*) FROM columnar.chunk WHERE storage_id=:storageid; 49 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_permissions.sql: -------------------------------------------------------------------------------- 1 | 2 | select current_user \gset 3 | 4 | create user columnar_user; 5 | grant all on schema public to columnar_user; 6 | 7 | \c - columnar_user 8 | 9 | create table columnar_permissions(i int) using columnar; 10 | insert into columnar_permissions values(1); 11 | alter table columnar_permissions add column j int; 12 | insert into columnar_permissions values(2,20); 13 | vacuum columnar_permissions; 14 | truncate columnar_permissions; 15 | drop table columnar_permissions; 16 | 17 | \c - :current_user 18 | 19 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_rollback.sql: -------------------------------------------------------------------------------- 1 | -- 2 | -- Testing we handle rollbacks properly 3 | -- 4 | 5 | CREATE TABLE t(a int, b int) USING columnar; 6 | 7 | CREATE VIEW t_stripes AS 8 | SELECT * FROM columnar.stripe a, pg_class b 9 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname = 't'; 10 | 11 | BEGIN; 12 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; 13 | ROLLBACK; 14 | SELECT count(*) FROM t; 15 | 16 | select 17 | version_major, version_minor, reserved_stripe_id, reserved_row_number 18 | from columnar_test_helpers.columnar_storage_info('t'); 19 | 20 | -- check stripe metadata also have been rolled-back 21 | SELECT count(*) FROM t_stripes; 22 | 23 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; 24 | SELECT count(*) FROM t; 25 | 26 | SELECT count(*) FROM t_stripes; 27 | 28 | -- savepoint rollback 29 | BEGIN; 30 | SAVEPOINT s0; 31 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; 32 | SELECT count(*) FROM t; -- force flush 33 | SAVEPOINT s1; 34 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; 35 | 36 | select 37 | version_major, version_minor, reserved_stripe_id, reserved_row_number 38 | from columnar_test_helpers.columnar_storage_info('t'); 39 | 40 | SELECT count(*) FROM t; 41 | ROLLBACK TO SAVEPOINT s1; 42 | SELECT count(*) FROM t; 43 | ROLLBACK TO SAVEPOINT s0; 44 | SELECT count(*) FROM t; 45 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i; 46 | COMMIT; 47 | 48 | select 49 | version_major, version_minor, reserved_stripe_id, reserved_row_number 50 | from columnar_test_helpers.columnar_storage_info('t'); 51 | 52 | SELECT count(*) FROM t; 53 | 54 | SELECT count(*) FROM t_stripes; 55 | 56 | DROP TABLE t; 57 | DROP VIEW t_stripes; 58 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_upsert.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE upsert_test ( 2 | i INT, 3 | t TEXT 4 | ) USING columnar; 5 | 6 | CREATE UNIQUE INDEX ON upsert_test (t); 7 | 8 | INSERT INTO upsert_test (i, t) VALUES (1, 'hello'); 9 | INSERT INTO upsert_test (i, t) VALUES (2, 'world'); 10 | 11 | -- should work, then roll back 12 | BEGIN; 13 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'foo'; 14 | SELECT * FROM upsert_test; 15 | ROLLBACK; 16 | 17 | SELECT * FROM upsert_test; 18 | 19 | -- should take affect immediately 20 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'bar'; 21 | SELECT * FROM upsert_test; 22 | 23 | -- should work as expected 24 | BEGIN; 25 | INSERT INTO upsert_test (t) VALUES ( 'world' ) ON CONFLICT (t) DO UPDATE SET t = 'foo'; 26 | SELECT * FROM upsert_test; 27 | COMMIT; 28 | 29 | -- should also work as expected, a select can mask a bug 30 | DELETE FROM upsert_test; 31 | BEGIN; 32 | INSERT INTO upsert_test (i, t) VALUES (1, 'hello'); 33 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'bar'; 34 | COMMIT; 35 | 36 | SELECT * FROM upsert_test; 37 | 38 | DROP TABLE upsert_test; 39 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_vectorization.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE t (id int, ts timestamp) USING columnar; 2 | INSERT INTO t SELECT id, now() + id * '1 day'::interval FROM generate_series(1, 100000) id; 3 | EXPLAIN (costs off) SELECT * FROM t WHERE ts between '2026-01-01'::timestamp and '2026-02-01'::timestamp; 4 | DROP TABLE t; 5 | 6 | CREATE TABLE t (id int, ts timestamptz) USING columnar; 7 | INSERT INTO t SELECT id, now() + id * '1 day'::interval FROM generate_series(1, 100000) id; 8 | EXPLAIN (costs off) SELECT * FROM t WHERE ts between '2026-01-01'::timestamptz and '2026-02-01'::timestamptz; 9 | DROP TABLE t; 10 | -------------------------------------------------------------------------------- /columnar/src/test/regress/sql/columnar_zstd.sql: -------------------------------------------------------------------------------- 1 | SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset 2 | \if :zstd_supported 3 | \else 4 | \q 5 | \endif 6 | 7 | CREATE SCHEMA am_zstd; 8 | SET search_path TO am_zstd; 9 | 10 | SET columnar.compression TO 'zstd'; 11 | CREATE TABLE test_zstd (a int, b text, c int) USING columnar; 12 | 13 | INSERT INTO test_zstd SELECT i % 1000, (i % 10)::text, 4 FROM generate_series(1, 10000) i; 14 | SELECT count(*) FROM test_zstd; 15 | 16 | INSERT INTO test_zstd SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 11000) i; 17 | SELECT count(*) FROM test_zstd; 18 | 19 | CREATE TABLE test_none (LIKE test_zstd) USING columnar; 20 | INSERT INTO test_none SELECT * FROM test_zstd; 21 | 22 | SELECT DISTINCT * FROM test_zstd ORDER BY a, b, c LIMIT 5; 23 | 24 | VACUUM FULL test_zstd; 25 | 26 | -- Hydra: we need to get `data_length` from stripe metadata information to compare compression 27 | -- rather than calling pg_relation_size function 28 | 29 | SELECT data_length AS size_comp_level_default FROM columnar.stripe WHERE storage_id = ( 30 | SELECT storage_id from columnar_test_helpers.columnar_storage_info('test_zstd')) \gset 31 | 32 | -- change compression level 33 | SELECT columnar.alter_columnar_table_set('test_zstd', compression_level => 19); 34 | VACUUM FULL test_zstd; 35 | 36 | SELECT data_length AS size_comp_level_19 FROM columnar.stripe WHERE storage_id = ( 37 | SELECT storage_id from columnar_test_helpers.columnar_storage_info('test_zstd')) \gset 38 | 39 | -- verify that higher compression level compressed better 40 | SELECT :size_comp_level_default > :size_comp_level_19 AS size_changed; 41 | 42 | -- compare compression rate to pglz 43 | SET columnar.compression TO 'pglz'; 44 | CREATE TABLE test_pglz (LIKE test_zstd) USING columnar; 45 | INSERT INTO test_pglz SELECT * FROM test_zstd; 46 | 47 | SELECT pg_relation_size('test_pglz') AS size_pglz \gset 48 | 49 | -- verify that zstd compressed better than pglz 50 | SELECT :size_pglz > :size_comp_level_default; 51 | 52 | -- Other operations 53 | 54 | ANALYZE test_zstd; 55 | SELECT count(DISTINCT test_zstd.*) FROM test_zstd; 56 | 57 | TRUNCATE test_zstd; 58 | 59 | SELECT count(DISTINCT test_zstd.*) FROM test_zstd; 60 | 61 | SET client_min_messages TO WARNING; 62 | DROP SCHEMA am_zstd CASCADE; 63 | -------------------------------------------------------------------------------- /columnar/vendor/README.md: -------------------------------------------------------------------------------- 1 | # Updating vendored dependencies 2 | 3 | ```bash 4 | rm -rf safestringlib 5 | git clone https://github.com/intel/safestringlib 6 | rm -rf safestringlib/{.git,unittests} 7 | git add safestringlib/ 8 | git commit -m "Update safestringlib" 9 | git cherry-pick -x dc2a371d9f8b28ad0e68c5230bb998b4463d8131 10 | ``` 11 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2014-2018 Intel Corporation 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | ================================================================================ 24 | 25 | Copyright (C) 2012, 2013 Cisco Systems 26 | All rights reserved. 27 | 28 | Permission is hereby granted, free of charge, to any person 29 | obtaining a copy of this software and associated documentation 30 | files (the "Software"), to deal in the Software without 31 | restriction, including without limitation the rights to use, 32 | copy, modify, merge, publish, distribute, sublicense, and/or 33 | sell copies of the Software, and to permit persons to whom the 34 | Software is furnished to do so, subject to the following 35 | conditions: 36 | 37 | The above copyright notice and this permission notice shall be 38 | included in all copies or substantial portions of the Software. 39 | 40 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 41 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 42 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 43 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 44 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 45 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 46 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 47 | OTHER DEALINGS IN THE SOFTWARE. 48 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/LICENSE©ING.txt: -------------------------------------------------------------------------------- 1 | Safe C Library 2 | 3 | Copyright (c) 2014-2016, Intel Corporation. All rights reserved. 4 | 5 | * Permission is hereby granted, free of charge, to any person 6 | * obtaining a copy of this software and associated documentation 7 | * files (the "Software"), to deal in the Software without 8 | * restriction, including without limitation the rights to use, 9 | * copy, modify, merge, publish, distribute, sublicense, and/or 10 | * sell copies of the Software, and to permit persons to whom the 11 | * Software is furnished to do so, subject to the following 12 | * conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | * OTHER DEALINGS IN THE SOFTWARE. 25 | 26 | ================================================================================ 27 | 28 | Copyright (C) 2012, 2013 Cisco Systems 29 | All rights reserved. 30 | 31 | Permission is hereby granted, free of charge, to any person 32 | obtaining a copy of this software and associated documentation 33 | files (the "Software"), to deal in the Software without 34 | restriction, including without limitation the rights to use, 35 | copy, modify, merge, publish, distribute, sublicense, and/or 36 | sell copies of the Software, and to permit persons to whom the 37 | Software is furnished to do so, subject to the following 38 | conditions: 39 | 40 | The above copyright notice and this permission notice shall be 41 | included in all copies or substantial portions of the Software. 42 | 43 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 44 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 45 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 46 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 47 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 48 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 49 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 50 | OTHER DEALINGS IN THE SOFTWARE. 51 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/README.md: -------------------------------------------------------------------------------- 1 | # safestringlib 2 | The Secure Development Lifecycle (SDL) recommends banning certain C Library functions because they directly contribute 3 | to security vulnerabilities such as buffer overflows. However routines for the manipulation of strings and memory buffers 4 | are common in software and firmware, and are essential to accomplish certain programming tasks. Safer replacements for 5 | these functions that avoid or prevent serious security vulnerabilities (e.g. buffer overflows, string format attacks, 6 | conversion overflows/underflows, etc.) are available in the SafeString Library. 7 | 8 | This library includes routines for safe string operations (like strcpy) and memory routines (like memcpy) that are 9 | recommended for Linux/Android operating systems, and will also work for Windows. This library is especially useful for 10 | cross-platform situations where one library for these routines is preferred. 11 | 12 | The Safe String Library is based on the Safe C Library by Cisco, and includes replacement C Library functions for the SDL 13 | banned functions, as well as a number of additional useful routines that are also susceptible to buffer overflows. This 14 | library continues to be made available under the MIT Open Source License. 15 | 16 | Cisco's Safe C Library was extended by Intel's Security Center of Excellence (SeCoE) to add additional routines, and 17 | include additional unit tests. 18 | 19 | LIST OF PRIMARY FUNCTIONS: 20 | ----------------------------- 21 | * memcmp_s() 22 | * memcpy_s() 23 | * memmove_s() 24 | * memset_s() 25 | * memzero_s() 26 | 27 | * stpcpy_s() 28 | * stpncpy_s() 29 | * strcat_s() 30 | * strcpy_s() 31 | * strcspn_s() 32 | * strncat_s() 33 | * strncpy_s() 34 | * strnlen_s() 35 | * strpbrk_s() 36 | * strspn_s() 37 | * strstr_s() 38 | * strtok_s() 39 | 40 | * wcpcpy_s() 41 | * wcscat_s() 42 | * wcscpy_s() 43 | * wcsncat_s() 44 | * wcsnlen_s() 45 | * wmemcmp_s() 46 | * wmemcpy_s() 47 | * wmemmove_s() 48 | * wmemset_s() 49 | 50 | 51 | LIST OF ADDITIONAL STRING ROUTINES: 52 | ------------------------------------ 53 | * strcasecmp_s() 54 | * strcasestr_s() 55 | * strcmp_s() 56 | * strcmpfld_s() 57 | * strcpyfld_s() 58 | * strcpyfldin_s() 59 | * strcpyfldout_s() 60 | * strfirstchar_s() 61 | * strfirstdiff_s() 62 | * strfirstsmae_s() 63 | * strisalphanumeric_s() 64 | * strisascii_s() 65 | * strisdigit_s() 66 | * strishes_s() 67 | * strislowercase_s() 68 | * strismixedcase_s() 69 | * strispassword_s() 70 | * strisuppercase_s() 71 | * strlastchar_s() 72 | * strlastdiff_s() 73 | * strlastsame_s() 74 | * strljustify_s() 75 | * strnterminate_s() 76 | * strprefix_s() 77 | * stremovews_s() 78 | * strtolowercase_s() 79 | * strtouppercase_s() 80 | * strzero_s() 81 | 82 | 83 | PLANNED ENHANCEMENTS: 84 | ---------------------- 85 | - Add full sprintf_s() support 86 | - Add full sscanf_s() support 87 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/include/safe_lib.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * safe_lib.h -- Safe C Library 3 | * 4 | * October 2008, Bo Berry 5 | * Modified 2012, Jonathan Toppins 6 | * 7 | * Copyright (c) 2008-2013 by Cisco Systems, Inc 8 | * All rights reserved. 9 | * 10 | * Permission is hereby granted, free of charge, to any person 11 | * obtaining a copy of this software and associated documentation 12 | * files (the "Software"), to deal in the Software without 13 | * restriction, including without limitation the rights to use, 14 | * copy, modify, merge, publish, distribute, sublicense, and/or 15 | * sell copies of the Software, and to permit persons to whom the 16 | * Software is furnished to do so, subject to the following 17 | * conditions: 18 | * 19 | * The above copyright notice and this permission notice shall be 20 | * included in all copies or substantial portions of the Software. 21 | * 22 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 23 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 24 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 25 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 26 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 27 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | * OTHER DEALINGS IN THE SOFTWARE. 30 | *------------------------------------------------------------------ 31 | */ 32 | 33 | #ifndef __SAFE_LIB_H__ 34 | #define __SAFE_LIB_H__ 35 | 36 | #ifdef __cplusplus 37 | extern "C" { 38 | #endif 39 | 40 | #include "safe_types.h" 41 | #include "safe_lib_errno.h" 42 | 43 | /* C11 appendix K types - specific for bounds checking */ 44 | typedef size_t rsize_t; 45 | 46 | /* 47 | * We depart from the standard and allow memory and string operations to 48 | * have different max sizes. See the respective safe_mem_lib.h or 49 | * safe_str_lib.h files. 50 | */ 51 | /* #define RSIZE_MAX (~(rsize_t)0) - leave here for completeness */ 52 | 53 | typedef void (*constraint_handler_t) (const char * /* msg */, 54 | void * /* ptr */, 55 | errno_t /* error */); 56 | 57 | extern void abort_handler_s(const char *msg, void *ptr, errno_t error); 58 | extern void ignore_handler_s(const char *msg, void *ptr, errno_t error); 59 | 60 | #define sl_default_handler ignore_handler_s 61 | 62 | #include "safe_mem_lib.h" 63 | #include "safe_str_lib.h" 64 | 65 | #ifdef __cplusplus 66 | } 67 | #endif 68 | #endif /* __SAFE_LIB_H__ */ 69 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/include/safe_types.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * safe_types.h - C99 std types & defs or Linux kernel equivalents 3 | * 4 | * March 2007, Bo Berry 5 | * Modified 2012, Jonathan Toppins 6 | * 7 | * Copyright (c) 2007-2013 by Cisco Systems, Inc 8 | * All rights reserved. 9 | * 10 | * Permission is hereby granted, free of charge, to any person 11 | * obtaining a copy of this software and associated documentation 12 | * files (the "Software"), to deal in the Software without 13 | * restriction, including without limitation the rights to use, 14 | * copy, modify, merge, publish, distribute, sublicense, and/or 15 | * sell copies of the Software, and to permit persons to whom the 16 | * Software is furnished to do so, subject to the following 17 | * conditions: 18 | * 19 | * The above copyright notice and this permission notice shall be 20 | * included in all copies or substantial portions of the Software. 21 | * 22 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 23 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 24 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 25 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 26 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 27 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | * OTHER DEALINGS IN THE SOFTWARE. 30 | *------------------------------------------------------------------ 31 | */ 32 | 33 | #ifndef __SAFE_TYPES_H__ 34 | #define __SAFE_TYPES_H__ 35 | 36 | #ifdef __KERNEL__ 37 | /* linux kernel environment */ 38 | 39 | #include 40 | #include 41 | #include 42 | 43 | /* errno_t isn't defined in the kernel */ 44 | typedef int errno_t; 45 | 46 | #else 47 | 48 | #include 49 | 50 | /* For systems without sys/types.h, prefer to get size_t from stdlib.h */ 51 | /* some armcc environments don't have a sys/types.h in the environment */ 52 | #ifdef _USE_STDLIB 53 | #include 54 | #include 55 | #else 56 | #include 57 | #endif 58 | 59 | #include 60 | #include 61 | #include 62 | 63 | typedef int errno_t; 64 | 65 | #include 66 | 67 | #endif /* __KERNEL__ */ 68 | #endif /* __SAFE_TYPES_H__ */ 69 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/include/safe_types.h.in: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * safe_types.h - C99 std types & defs or Linux kernel equivalents 3 | * 4 | * March 2007, Bo Berry 5 | * Modified 2012, Jonathan Toppins 6 | * 7 | * Copyright (c) 2007-2013 by Cisco Systems, Inc 8 | * All rights reserved. 9 | * 10 | * Permission is hereby granted, free of charge, to any person 11 | * obtaining a copy of this software and associated documentation 12 | * files (the "Software"), to deal in the Software without 13 | * restriction, including without limitation the rights to use, 14 | * copy, modify, merge, publish, distribute, sublicense, and/or 15 | * sell copies of the Software, and to permit persons to whom the 16 | * Software is furnished to do so, subject to the following 17 | * conditions: 18 | * 19 | * The above copyright notice and this permission notice shall be 20 | * included in all copies or substantial portions of the Software. 21 | * 22 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 23 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 24 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 25 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 26 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 27 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 28 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 29 | * OTHER DEALINGS IN THE SOFTWARE. 30 | *------------------------------------------------------------------ 31 | */ 32 | 33 | #ifndef __SAFE_TYPES_H__ 34 | #define __SAFE_TYPES_H__ 35 | 36 | #ifdef __KERNEL__ 37 | /* linux kernel environment */ 38 | 39 | #include 40 | #include 41 | #include 42 | 43 | /* errno_t isn't defined in the kernel */ 44 | typedef int errno_t; 45 | 46 | #else 47 | 48 | #include 49 | @INSERT_SYS_TYPES_H@ 50 | @INSERT_INTTYPES_H@ 51 | @INSERT_STDINT_H@ 52 | @INSERT_ERRNO_H@ 53 | 54 | @FALLBACK_ERRNO_T@ 55 | 56 | @INSERT_BOOL_SUPPORT@ 57 | 58 | #endif /* __KERNEL__ */ 59 | #endif /* __SAFE_TYPES_H__ */ 60 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/include/snprintf_s.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * sprintf_s.h -- Safe Sprintf Interfaces 3 | * 4 | * August 2014, D Wheeler 5 | * 6 | * Copyright (c) 2014 by Intel Corp 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | #ifndef SPRINTF_S_H_ 32 | #define SPRINTF_S_H_ 33 | 34 | #include 35 | 36 | 37 | #define SNPRFNEGATE(x) (-1*(x)) 38 | 39 | 40 | 41 | int snprintf_s_i(char *dest, rsize_t dmax, const char *format, int a); 42 | int snprintf_s_si(char *dest, rsize_t dmax, const char *format, char *s, int a); 43 | int snprintf_s_l(char *dest, rsize_t dmax, const char *format, long a); 44 | int snprintf_s_sl(char *dest, rsize_t dmax, const char *format, char *s, long a); 45 | 46 | 47 | 48 | #endif /* SPRINTF_S_H_ */ 49 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/safeclib/abort_handler_s.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * abort_handler_s.c 3 | * 4 | * 2012, Jonathan Toppins 5 | * 6 | * Copyright (c) 2012 Cisco Systems 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #include "safeclib_private.h" 33 | 34 | /** 35 | * NAME 36 | * abort_handler_s 37 | * 38 | * SYNOPSIS 39 | * #include "safe_lib.h" 40 | * void abort_handler_s(const char *msg, void *ptr, errno_t error) 41 | * 42 | * DESCRIPTION 43 | * This function writes a message on the standard error stream in 44 | * an implementation-defined format. The message shall include the 45 | * string pointed to by msg. The abort_handler_s function then calls 46 | * the abort function. 47 | * 48 | * SPECIFIED IN 49 | * ISO/IEC JTC1 SC22 WG14 N1172, Programming languages, environments 50 | * and system software interfaces, Extensions to the C Library, 51 | * Part I: Bounds-checking interfaces 52 | * 53 | * INPUT PARAMETERS 54 | * msg Pointer to the message describing the error 55 | * 56 | * ptr Pointer to aassociated data. Can be NULL. 57 | * 58 | * error The error code encountered. 59 | * 60 | * RETURN VALUE 61 | * Does not return to caller. 62 | * 63 | * ALSO SEE 64 | * ignore_handler_s() 65 | * 66 | */ 67 | 68 | void abort_handler_s(const char *msg, void *ptr, errno_t error) 69 | { 70 | slprintf("ABORT CONSTRAINT HANDLER: (%u) %s\n", error, 71 | (msg) ? msg : "Null message"); 72 | slabort(); 73 | } 74 | EXPORT_SYMBOL(abort_handler_s) 75 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/safeclib/ignore_handler_s.c: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * ignore_handler_s.c 3 | * 4 | * 2012, Jonathan Toppins 5 | * 6 | * Copyright (c) 2012 Cisco Systems 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #include "safeclib_private.h" 33 | 34 | /** 35 | * NAME 36 | * ignore_handler_s 37 | * 38 | * SYNOPSIS 39 | * #include "safe_lib.h" 40 | * void ignore_handler_s(const char *msg, void *ptr, errno_t error) 41 | * 42 | * DESCRIPTION 43 | * This function simply returns to the caller. 44 | * 45 | * SPECIFIED IN 46 | * ISO/IEC JTC1 SC22 WG14 N1172, Programming languages, environments 47 | * and system software interfaces, Extensions to the C Library, 48 | * Part I: Bounds-checking interfaces 49 | * 50 | * INPUT PARAMETERS 51 | * msg Pointer to the message describing the error 52 | * 53 | * ptr Pointer to aassociated data. Can be NULL. 54 | * 55 | * error The error code encountered. 56 | * 57 | * RETURN VALUE 58 | * Returns no value. 59 | * 60 | * ALSO SEE 61 | * abort_handler_s() 62 | * 63 | */ 64 | 65 | void ignore_handler_s(const char *msg, void *ptr, errno_t error) 66 | { 67 | 68 | sldebug_printf("IGNORE CONSTRAINT HANDLER: (%u) %s\n", error, 69 | (msg) ? msg : "Null message"); 70 | return; 71 | } 72 | EXPORT_SYMBOL(ignore_handler_s) 73 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/safeclib/mem_primitives_lib.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * mem_primitives_lib.h - Unguarded Memory Copy Routines 3 | * 4 | * October 2008, Bo Berry 5 | * 6 | * Copyright (c) 2008-2011 by Cisco Systems, Inc 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #ifndef __MEM_PRIMITIVES_LIB_H__ 33 | #define __MEM_PRIMITIVES_LIB_H__ 34 | 35 | #include "safeclib_private.h" 36 | 37 | /* 38 | * These are prototypes for _unguarded_ memory routines. The caller must 39 | * validate all parameters prior to invocation. Useful for diagnostics 40 | * and system initialization processing. 41 | */ 42 | 43 | /* moves (handles overlap) memory */ 44 | extern void 45 | mem_prim_move(void *dest, const void *src, uint32_t length); 46 | 47 | 48 | /* uint8_t moves (handles overlap) memory */ 49 | extern void 50 | mem_prim_move8(uint8_t *dest, const uint8_t *src, uint32_t length); 51 | 52 | /* uint16_t moves (handles overlap) memory */ 53 | extern void 54 | mem_prim_move16(uint16_t *dest, const uint16_t *src, uint32_t length); 55 | 56 | /* uint32_t moves (handles overlap) memory */ 57 | extern void 58 | mem_prim_move32(uint32_t *dest, const uint32_t *src, uint32_t length); 59 | 60 | 61 | /* set bytes */ 62 | extern void 63 | mem_prim_set(void *dest, uint32_t dmax, uint8_t value); 64 | 65 | /* set uint16_ts */ 66 | extern void 67 | mem_prim_set16(uint16_t *dest, uint32_t dmax, uint16_t value); 68 | 69 | /* set uint32_ts */ 70 | extern void 71 | mem_prim_set32(uint32_t *dest, uint32_t dmax, uint32_t value); 72 | 73 | 74 | #endif /* __MEM_PRIMITIVES_LIB_H__ */ 75 | -------------------------------------------------------------------------------- /columnar/vendor/safestringlib/safeclib/safe_mem_constraint.h: -------------------------------------------------------------------------------- 1 | /*------------------------------------------------------------------ 2 | * safe_mem_constraint.h 3 | * 4 | * October 2008, Bo Berry 5 | * 6 | * Copyright (c) 2008, 2009 by Cisco Systems, Inc. 7 | * All rights reserved. 8 | * 9 | * Permission is hereby granted, free of charge, to any person 10 | * obtaining a copy of this software and associated documentation 11 | * files (the "Software"), to deal in the Software without 12 | * restriction, including without limitation the rights to use, 13 | * copy, modify, merge, publish, distribute, sublicense, and/or 14 | * sell copies of the Software, and to permit persons to whom the 15 | * Software is furnished to do so, subject to the following 16 | * conditions: 17 | * 18 | * The above copyright notice and this permission notice shall be 19 | * included in all copies or substantial portions of the Software. 20 | * 21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 28 | * OTHER DEALINGS IN THE SOFTWARE. 29 | *------------------------------------------------------------------ 30 | */ 31 | 32 | #ifndef __SAFE_MEM_CONSTRAINT_H__ 33 | #define __SAFE_MEM_CONSTRAINT_H__ 34 | 35 | #include "safeclib_private.h" 36 | 37 | /* 38 | * Function used by the libraries to invoke the registered 39 | * runtime-constraint handler. Always needed. 40 | */ 41 | extern void invoke_safe_mem_constraint_handler( 42 | const char *msg, 43 | void *ptr, 44 | errno_t error); 45 | 46 | #endif /* __SAFE_MEM_CONSTRAINT_H__ */ 47 | -------------------------------------------------------------------------------- /docker-bake.hcl: -------------------------------------------------------------------------------- 1 | variable "POSTGRES_REPO" { 2 | default = "ghcr.io/hydradatabase/hydra" 3 | } 4 | 5 | variable "SPILO_REPO" { 6 | default = "011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo" 7 | } 8 | 9 | variable "SPILO_VERSION" { 10 | default = "3.2-p2" 11 | } 12 | 13 | variable "POSTGRES_BASE_VERSION" { 14 | default = "14" 15 | } 16 | 17 | variable "SPILO_POSTGRES_VERSION" { 18 | default = "14" 19 | } 20 | 21 | variable "SPILO_POSTGRES_OLD_VERSIONS" { 22 | default = "13" 23 | } 24 | 25 | group "default" { 26 | targets = ["postgres", "spilo"] 27 | } 28 | 29 | target "shared" { 30 | platforms = [ 31 | "linux/amd64", 32 | "linux/arm64" 33 | ] 34 | 35 | args = { 36 | TIMESTAMP = "${timestamp()}" 37 | } 38 | } 39 | 40 | target "postgres" { 41 | inherits = ["shared"] 42 | 43 | contexts = { 44 | columnar = "target:columnar_${POSTGRES_BASE_VERSION}" 45 | postgres_base = "docker-image://postgres:${POSTGRES_BASE_VERSION}-bookworm" 46 | } 47 | 48 | args = { 49 | POSTGRES_BASE_VERSION = "${POSTGRES_BASE_VERSION}" 50 | } 51 | 52 | tags = [ 53 | "${POSTGRES_REPO}:latest", 54 | "${POSTGRES_REPO}:${POSTGRES_BASE_VERSION}" 55 | ] 56 | } 57 | 58 | target "spilo" { 59 | inherits = ["shared"] 60 | 61 | dockerfile = "Dockerfile.spilo" 62 | 63 | contexts = { 64 | spilo_base = "target:spilo_base" 65 | columnar_13 = "target:columnar_13" 66 | columnar_14 = "target:columnar_14" 67 | columnar_15 = "target:columnar_15" 68 | } 69 | 70 | args = { 71 | POSTGRES_BASE_VERSION = "${SPILO_POSTGRES_VERSION}" 72 | } 73 | 74 | tags = [ 75 | "${SPILO_REPO}:latest", 76 | "${SPILO_REPO}:${SPILO_VERSION}-latest" 77 | ] 78 | } 79 | 80 | target "spilo_base" { 81 | inherits = ["shared"] 82 | 83 | context = "https://github.com/zalando/spilo.git#${SPILO_VERSION}:postgres-appliance" 84 | 85 | args = { 86 | TIMESCALEDB = "" 87 | PGVERSION = "${SPILO_POSTGRES_VERSION}" 88 | PGOLDVERSIONS = "${SPILO_POSTGRES_OLD_VERSIONS}" 89 | } 90 | } 91 | 92 | target "columnar" { 93 | inherits = ["shared"] 94 | context = "columnar" 95 | target = "output" 96 | } 97 | 98 | target "columnar_13" { 99 | inherits = ["columnar"] 100 | 101 | args = { 102 | POSTGRES_BASE_VERSION = 13 103 | } 104 | } 105 | 106 | target "columnar_14" { 107 | inherits = ["columnar"] 108 | 109 | args = { 110 | POSTGRES_BASE_VERSION = 14 111 | } 112 | } 113 | 114 | target "columnar_15" { 115 | inherits = ["columnar"] 116 | 117 | args = { 118 | POSTGRES_BASE_VERSION = 15 119 | } 120 | } 121 | 122 | target "columnar_16" { 123 | inherits = ["columnar"] 124 | 125 | args = { 126 | POSTGRES_BASE_VERSION = 16 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | hydra: 3 | container_name: hydra 4 | image: ghcr.io/hydradatabase/hydra:latest 5 | ports: 6 | - ${POSTGRES_PORT}:5432 7 | environment: 8 | POSTGRES_USER: ${POSTGRES_USER} 9 | POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} 10 | volumes: 11 | - ./volumes/db:/var/lib/postgresql/data 12 | - ./files/postgres/postgresql.conf:/etc/postgresql/postgresql.conf 13 | command: postgres -c 'config_file=/etc/postgresql/postgresql.conf' 14 | -------------------------------------------------------------------------------- /files/postgres/docker-entrypoint-initdb.d/columnar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL 6 | CREATE EXTENSION IF NOT EXISTS columnar; 7 | ALTER EXTENSION columnar UPDATE; 8 | ALTER DATABASE "${POSTGRES_DB}" SET default_table_access_method = 'columnar'; 9 | EOSQL 10 | -------------------------------------------------------------------------------- /files/spilo/postgres-appliance/scripts/multicorn/after-create.sql: -------------------------------------------------------------------------------- 1 | GRANT USAGE ON FOREIGN DATA WRAPPER multicorn TO admin; 2 | -------------------------------------------------------------------------------- /files/spilo/postgres-appliance/scripts/mysql_fdw/after-create.sql: -------------------------------------------------------------------------------- 1 | GRANT USAGE ON FOREIGN DATA WRAPPER mysql_fdw TO admin; 2 | -------------------------------------------------------------------------------- /files/spilo/postgres-appliance/scripts/parquet_s3_fdw/after-create.sql: -------------------------------------------------------------------------------- 1 | GRANT USAGE ON FOREIGN DATA WRAPPER parquet_s3_fdw TO admin; 2 | -------------------------------------------------------------------------------- /third-party/pgxman/pgxman_13.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | postgres: 3 | version: "13" 4 | extensions: 5 | - name: "multicorn" 6 | version: "2.4.0+b68b75c" 7 | - name: "mysql_fdw" 8 | version: "2.9.1" 9 | - name: "parquet_s3_fdw" 10 | version: "1.0.0+5298b7f" 11 | - name: "pg_ivm" 12 | version: "1.7.0" 13 | - name: "pgvector" 14 | version: "0.5.1" 15 | - name: "pg_hint_plan" 16 | version: "1.3.9" 17 | -------------------------------------------------------------------------------- /third-party/pgxman/pgxman_13_spilo.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | postgres: 3 | version: "13" 4 | extensions: 5 | - name: "multicorn" 6 | version: "2.4.0+b68b75c" 7 | - name: "mysql_fdw" 8 | version: "2.9.1" 9 | - name: "parquet_s3_fdw" 10 | version: "1.0.0+5298b7f" 11 | - name: "pg_ivm" 12 | version: "1.7.0" 13 | - name: "pgvector" 14 | version: "0.5.1" 15 | overwrite: true 16 | - name: "pg_hint_plan" 17 | version: "1.3.9" 18 | -------------------------------------------------------------------------------- /third-party/pgxman/pgxman_14.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | postgres: 3 | version: "14" 4 | extensions: 5 | - name: "multicorn" 6 | version: "2.4.0+b68b75c" 7 | - name: "mysql_fdw" 8 | version: "2.9.1" 9 | - name: "parquet_s3_fdw" 10 | version: "1.0.0+5298b7f" 11 | - name: "pg_ivm" 12 | version: "1.7.0" 13 | - name: "pgvector" 14 | version: "0.5.1" 15 | - name: "pg_hint_plan" 16 | version: "1.4.2" 17 | - name: "wrappers" 18 | version: "0.2.0" 19 | - name: "pgsodium" 20 | version: "3.1.9" 21 | - name: "supabase_vault" 22 | version: "0.2.9" 23 | -------------------------------------------------------------------------------- /third-party/pgxman/pgxman_14_spilo.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | postgres: 3 | version: "14" 4 | extensions: 5 | - name: "multicorn" 6 | version: "2.4.0+b68b75c" 7 | overwrite: true 8 | - name: "mysql_fdw" 9 | version: "2.9.1" 10 | - name: "parquet_s3_fdw" 11 | version: "1.0.0+5298b7f" 12 | overwrite: true 13 | - name: "pg_ivm" 14 | version: "1.7.0" 15 | - name: "pgvector" 16 | version: "0.5.1" 17 | overwrite: true 18 | - name: "pg_hint_plan" 19 | version: "1.4.2" 20 | - name: "wrappers" 21 | version: "0.2.0" 22 | - name: "pgsodium" 23 | version: "3.1.9" 24 | - name: "supabase_vault" 25 | version: "0.2.9" 26 | -------------------------------------------------------------------------------- /third-party/pgxman/pgxman_15.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | postgres: 3 | version: "15" 4 | extensions: 5 | - name: "multicorn" 6 | version: "2.4.0+b68b75c" 7 | - name: "mysql_fdw" 8 | version: "2.9.1" 9 | - name: "parquet_s3_fdw" 10 | version: "1.0.0+5298b7f" 11 | - name: "pg_ivm" 12 | version: "1.7.0" 13 | - name: "pgvector" 14 | version: "0.5.1" 15 | - name: "pg_hint_plan" 16 | version: "1.5.1" 17 | - name: "wrappers" 18 | version: "0.2.0" 19 | - name: "pgsodium" 20 | version: "3.1.9" 21 | - name: "supabase_vault" 22 | version: "0.2.9" 23 | -------------------------------------------------------------------------------- /third-party/pgxman/pgxman_16.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | postgres: 3 | version: "16" 4 | extensions: 5 | - name: "mysql_fdw" 6 | version: "2.9.1" 7 | - name: "pg_ivm" 8 | version: "1.7.0" 9 | - name: "pgvector" 10 | version: "0.5.1" 11 | - name: "pg_hint_plan" 12 | version: "1.6.0" 13 | - name: "wrappers" 14 | version: "0.2.0" 15 | - name: "pgsodium" 16 | version: "3.1.9" 17 | - name: "supabase_vault" 18 | version: "0.2.9" 19 | --------------------------------------------------------------------------------