├── .env.example
├── .github
├── ISSUE_TEMPLATE
│ ├── 1.general_bug_report.yaml
│ └── config.yml
├── PULL_REQUEST_TEMPLATE.md
└── workflows
│ ├── benchmark.yaml
│ ├── build_spilo.yaml
│ └── ci.yaml
├── .gitignore
├── .golangci_lint_version
├── .images
├── FasterthanPG.png
├── ReadmeBenchmarks.png
└── header.png
├── BENCHMARKS.md
├── CHANGELOG.md
├── DEVELOPERS.md
├── Dockerfile
├── Dockerfile.spilo
├── HYDRA_PROD_VER
├── LICENSE
├── Makefile
├── README.md
├── VERSION
├── acceptance
├── .golangci.yml
├── fixtures
│ └── mysql.sql
├── go.mod
├── go.sum
├── postgres
│ └── main_test.go
├── shared
│ ├── cases.go
│ ├── pg.go
│ ├── runner.go
│ └── shared.go
└── spilo
│ └── main_test.go
├── bin
└── autolink_changelog.rb
├── columnar
├── .gitattributes
├── .gitignore
├── .ignore
├── Dockerfile
├── LICENSE
├── Makefile
├── Makefile.global.in
├── NOTICE
├── README.md
├── aclocal.m4
├── autogen.sh
├── cgmanifest.json
├── config
│ ├── config.guess
│ └── general.m4
├── configure
├── configure.in
├── docker-build.sh
├── prep_buildtree
├── src
│ ├── backend
│ │ ├── .gitignore
│ │ └── columnar
│ │ │ ├── .gitattributes
│ │ │ ├── .gitignore
│ │ │ ├── Makefile
│ │ │ ├── README.md
│ │ │ ├── columnar.c
│ │ │ ├── columnar.control
│ │ │ ├── columnar_cache.c
│ │ │ ├── columnar_compression.c
│ │ │ ├── columnar_customscan.c
│ │ │ ├── columnar_debug.c
│ │ │ ├── columnar_indexscan.c
│ │ │ ├── columnar_metadata.c
│ │ │ ├── columnar_planner_hook.c
│ │ │ ├── columnar_read_state_cache.c
│ │ │ ├── columnar_reader.c
│ │ │ ├── columnar_storage.c
│ │ │ ├── columnar_tableam.c
│ │ │ ├── columnar_writer.c
│ │ │ ├── mod.c
│ │ │ ├── safeclib
│ │ │ ├── abort_handler_s.c
│ │ │ ├── ignore_handler_s.c
│ │ │ ├── mem_primitives_lib.c
│ │ │ ├── mem_primitives_lib.h
│ │ │ ├── memcmp16_s.c
│ │ │ ├── memcmp32_s.c
│ │ │ ├── memcmp_s.c
│ │ │ ├── memcpy16_s.c
│ │ │ ├── memcpy32_s.c
│ │ │ ├── memcpy_s.c
│ │ │ ├── memmove16_s.c
│ │ │ ├── memmove32_s.c
│ │ │ ├── memmove_s.c
│ │ │ ├── memset16_s.c
│ │ │ ├── memset32_s.c
│ │ │ ├── memset_s.c
│ │ │ ├── memzero16_s.c
│ │ │ ├── memzero32_s.c
│ │ │ ├── memzero_s.c
│ │ │ ├── safe_mem_constraint.c
│ │ │ ├── safe_mem_constraint.h
│ │ │ ├── safe_str_constraint.c
│ │ │ ├── safe_str_constraint.h
│ │ │ ├── safeclib_private.h
│ │ │ ├── snprintf_support.c
│ │ │ ├── stpcpy_s.c
│ │ │ ├── stpncpy_s.c
│ │ │ ├── strcasecmp_s.c
│ │ │ ├── strcasestr_s.c
│ │ │ ├── strcat_s.c
│ │ │ ├── strcmp_s.c
│ │ │ ├── strcmpfld_s.c
│ │ │ ├── strcpy_s.c
│ │ │ ├── strcpyfld_s.c
│ │ │ ├── strcpyfldin_s.c
│ │ │ ├── strcpyfldout_s.c
│ │ │ ├── strcspn_s.c
│ │ │ ├── strfirstchar_s.c
│ │ │ ├── strfirstdiff_s.c
│ │ │ ├── strfirstsame_s.c
│ │ │ ├── strisalphanumeric_s.c
│ │ │ ├── strisascii_s.c
│ │ │ ├── strisdigit_s.c
│ │ │ ├── strishex_s.c
│ │ │ ├── strislowercase_s.c
│ │ │ ├── strismixedcase_s.c
│ │ │ ├── strispassword_s.c
│ │ │ ├── strisuppercase_s.c
│ │ │ ├── strlastchar_s.c
│ │ │ ├── strlastdiff_s.c
│ │ │ ├── strlastsame_s.c
│ │ │ ├── strljustify_s.c
│ │ │ ├── strncat_s.c
│ │ │ ├── strncpy_s.c
│ │ │ ├── strnlen_s.c
│ │ │ ├── strnterminate_s.c
│ │ │ ├── strpbrk_s.c
│ │ │ ├── strprefix_s.c
│ │ │ ├── strremovews_s.c
│ │ │ ├── strspn_s.c
│ │ │ ├── strstr_s.c
│ │ │ ├── strtok_s.c
│ │ │ ├── strtolowercase_s.c
│ │ │ ├── strtouppercase_s.c
│ │ │ ├── strzero_s.c
│ │ │ ├── wcpcpy_s.c
│ │ │ ├── wcscat_s.c
│ │ │ ├── wcscpy_s.c
│ │ │ ├── wcsncat_s.c
│ │ │ ├── wcsncpy_s.c
│ │ │ ├── wcsnlen_s.c
│ │ │ ├── wmemcmp_s.c
│ │ │ ├── wmemcpy_s.c
│ │ │ ├── wmemmove_s.c
│ │ │ └── wmemset_s.c
│ │ │ ├── sql
│ │ │ ├── columnar--10.0-1--10.0-2.sql
│ │ │ ├── columnar--10.0-2--10.0-3.sql
│ │ │ ├── columnar--10.0-3--10.1-1.sql
│ │ │ ├── columnar--10.1-1--10.2-1.sql
│ │ │ ├── columnar--10.2-1--10.2-2.sql
│ │ │ ├── columnar--10.2-2--10.2-3.sql
│ │ │ ├── columnar--10.2-3--10.2-4.sql
│ │ │ ├── columnar--10.2-4--11.1-1.sql
│ │ │ ├── columnar--11.1-1--11.1-2.sql
│ │ │ ├── columnar--11.1-10--11.1-11.sql
│ │ │ ├── columnar--11.1-11--11.1-12.sql
│ │ │ ├── columnar--11.1-2--11.1-3.sql
│ │ │ ├── columnar--11.1-3--11.1-4.sql
│ │ │ ├── columnar--11.1-4--11.1-5.sql
│ │ │ ├── columnar--11.1-5--11.1-6.sql
│ │ │ ├── columnar--11.1-6--11.1-7.sql
│ │ │ ├── columnar--11.1-7--11.1-8.sql
│ │ │ ├── columnar--11.1-8--11.1-9.sql
│ │ │ ├── columnar--11.1-9--11.1-10.sql
│ │ │ ├── columnar--9.5-1--10.0-1.sql
│ │ │ ├── columnar--9.5-1.sql
│ │ │ ├── downgrades
│ │ │ │ ├── columnar--10.0-1--9.5-1.sql
│ │ │ │ ├── columnar--10.0-2--10.0-1.sql
│ │ │ │ ├── columnar--10.1-1--10.0-3.sql
│ │ │ │ ├── columnar--10.2-1--10.1-1.sql
│ │ │ │ ├── columnar--10.2-2--10.2-1.sql
│ │ │ │ ├── columnar--10.2-3--10.2-2.sql
│ │ │ │ ├── columnar--10.2-4--10.2-3.sql
│ │ │ │ ├── columnar--11.1-1--10.2-4.sql
│ │ │ │ ├── columnar--11.1-2--11.1-1.sql
│ │ │ │ ├── columnar--11.1-3--11.1-2.sql
│ │ │ │ ├── columnar--11.1-4--11.1-3.sql
│ │ │ │ └── columnar--11.1-5--11.1-4.sql
│ │ │ └── udfs
│ │ │ │ ├── alter_columnar_table_reset
│ │ │ │ ├── 10.0-1.sql
│ │ │ │ ├── 11.1-2.sql
│ │ │ │ ├── 11.1-3.sql
│ │ │ │ └── latest.sql
│ │ │ │ ├── alter_columnar_table_set
│ │ │ │ ├── 10.0-1.sql
│ │ │ │ ├── 11.1-2.sql
│ │ │ │ ├── 11.1-3.sql
│ │ │ │ └── latest.sql
│ │ │ │ ├── alter_table_set_access_method
│ │ │ │ ├── 11.1-11.sql
│ │ │ │ ├── 11.1-12.sql
│ │ │ │ ├── 11.1-2.sql
│ │ │ │ ├── 11.1-4.sql
│ │ │ │ ├── 11.1-5.sql
│ │ │ │ ├── 11.1-8.sql
│ │ │ │ └── latest.sql
│ │ │ │ ├── columnar_ensure_am_depends_catalog
│ │ │ │ ├── 10.2-4.sql
│ │ │ │ └── latest.sql
│ │ │ │ ├── columnar_ensure_objects_exist
│ │ │ │ ├── 10.0-1.sql
│ │ │ │ └── latest.sql
│ │ │ │ ├── columnar_handler
│ │ │ │ ├── 10.0-1.sql
│ │ │ │ └── latest.sql
│ │ │ │ ├── create_table_row_mask
│ │ │ │ ├── 11.1-5.sql
│ │ │ │ └── latest.sql
│ │ │ │ ├── downgrade_columnar_storage
│ │ │ │ ├── 10.2-1.sql
│ │ │ │ └── latest.sql
│ │ │ │ ├── upgrade_columnar_storage
│ │ │ │ ├── 10.2-1.sql
│ │ │ │ └── latest.sql
│ │ │ │ └── vacuum
│ │ │ │ ├── 11.1-10.sql
│ │ │ │ ├── 11.1-6.sql
│ │ │ │ ├── 11.1-7.sql
│ │ │ │ └── latest.sql
│ │ │ ├── vectorization
│ │ │ ├── columnar_vector_execution.c
│ │ │ ├── columnar_vector_types.c
│ │ │ ├── nodes
│ │ │ │ └── columnar_aggregator_node.c
│ │ │ └── types
│ │ │ │ ├── aggregates.c
│ │ │ │ ├── date.c
│ │ │ │ ├── int.c
│ │ │ │ └── numeric.c
│ │ │ ├── write_state_interface.c
│ │ │ ├── write_state_management.c
│ │ │ └── write_state_row_mask.c
│ ├── include
│ │ ├── .gitignore
│ │ ├── citus_config.h.in
│ │ ├── citus_version.h.in
│ │ ├── columnar
│ │ │ ├── columnar.h
│ │ │ ├── columnar_compression.h
│ │ │ ├── columnar_customscan.h
│ │ │ ├── columnar_indexscan.h
│ │ │ ├── columnar_metadata.h
│ │ │ ├── columnar_storage.h
│ │ │ ├── columnar_tableam.h
│ │ │ ├── columnar_version_compat.h
│ │ │ ├── columnar_write_state_row_mask.h
│ │ │ ├── utils
│ │ │ │ └── listutils.h
│ │ │ └── vectorization
│ │ │ │ ├── columnar_vector_execution.h
│ │ │ │ ├── columnar_vector_types.h
│ │ │ │ ├── nodes
│ │ │ │ └── columnar_aggregator_node.h
│ │ │ │ └── types
│ │ │ │ ├── numeric.h
│ │ │ │ └── types.h
│ │ ├── pg_version_compat.h
│ │ └── pg_version_constants.h
│ └── test
│ │ └── regress
│ │ ├── .gitignore
│ │ ├── Makefile
│ │ ├── README.md
│ │ ├── columnar_regression.conf
│ │ ├── columnar_schedule
│ │ ├── data
│ │ ├── array_types.csv
│ │ ├── contestants.1.csv
│ │ ├── contestants.2.csv
│ │ ├── datetime_types.csv
│ │ ├── enum_and_composite_types.csv
│ │ ├── null_values.csv
│ │ ├── other_types.csv
│ │ └── range_types.csv
│ │ ├── expected
│ │ ├── .gitignore
│ │ ├── columnar_aggregates.out
│ │ ├── columnar_aggregates_1.out
│ │ ├── columnar_aggregates_2.out
│ │ ├── columnar_alter.out
│ │ ├── columnar_alter_set_type.out
│ │ ├── columnar_alter_table_set_access_method.out
│ │ ├── columnar_analyze.out
│ │ ├── columnar_cache.out
│ │ ├── columnar_cache_1.out
│ │ ├── columnar_chunk_filtering.out
│ │ ├── columnar_clean.out
│ │ ├── columnar_create.out
│ │ ├── columnar_cursor.out
│ │ ├── columnar_customindex.out
│ │ ├── columnar_customindex_1.out
│ │ ├── columnar_drop.out
│ │ ├── columnar_empty.out
│ │ ├── columnar_fallback_scan.out
│ │ ├── columnar_first_row_number.out
│ │ ├── columnar_indexes.out
│ │ ├── columnar_insert.out
│ │ ├── columnar_insert_1.out
│ │ ├── columnar_insert_2.out
│ │ ├── columnar_join.out
│ │ ├── columnar_lz4.out
│ │ ├── columnar_matview.out
│ │ ├── columnar_memory.out
│ │ ├── columnar_partitioning.out
│ │ ├── columnar_paths.out
│ │ ├── columnar_paths_1.out
│ │ ├── columnar_paths_2.out
│ │ ├── columnar_permissions.out
│ │ ├── columnar_query.out
│ │ ├── columnar_recursive.out
│ │ ├── columnar_rollback.out
│ │ ├── columnar_tableoptions.out
│ │ ├── columnar_test_helpers.out
│ │ ├── columnar_transactions.out
│ │ ├── columnar_trigger.out
│ │ ├── columnar_truncate.out
│ │ ├── columnar_types_without_comparison.out
│ │ ├── columnar_update_delete.out
│ │ ├── columnar_upsert.out
│ │ ├── columnar_vacuum.out
│ │ ├── columnar_vacuum_udf.out
│ │ ├── columnar_vacuum_udf_1.out
│ │ ├── columnar_vectorization.out
│ │ └── columnar_zstd.out
│ │ ├── input
│ │ ├── columnar_copyto.source
│ │ ├── columnar_data_types.source
│ │ └── columnar_load.source
│ │ ├── output
│ │ ├── columnar_copyto.source
│ │ ├── columnar_data_types.source
│ │ └── columnar_load.source
│ │ └── sql
│ │ ├── .gitignore
│ │ ├── columnar_aggregates.sql
│ │ ├── columnar_alter.sql
│ │ ├── columnar_alter_set_type.sql
│ │ ├── columnar_alter_table_set_access_method.sql
│ │ ├── columnar_analyze.sql
│ │ ├── columnar_cache.sql
│ │ ├── columnar_chunk_filtering.sql
│ │ ├── columnar_clean.sql
│ │ ├── columnar_create.sql
│ │ ├── columnar_cursor.sql
│ │ ├── columnar_customindex.sql
│ │ ├── columnar_drop.sql
│ │ ├── columnar_empty.sql
│ │ ├── columnar_fallback_scan.sql
│ │ ├── columnar_first_row_number.sql
│ │ ├── columnar_indexes.sql
│ │ ├── columnar_insert.sql
│ │ ├── columnar_join.sql
│ │ ├── columnar_lz4.sql
│ │ ├── columnar_matview.sql
│ │ ├── columnar_memory.sql
│ │ ├── columnar_partitioning.sql
│ │ ├── columnar_paths.sql
│ │ ├── columnar_permissions.sql
│ │ ├── columnar_query.sql
│ │ ├── columnar_recursive.sql
│ │ ├── columnar_rollback.sql
│ │ ├── columnar_tableoptions.sql
│ │ ├── columnar_test_helpers.sql
│ │ ├── columnar_transactions.sql
│ │ ├── columnar_trigger.sql
│ │ ├── columnar_truncate.sql
│ │ ├── columnar_types_without_comparison.sql
│ │ ├── columnar_update_delete.sql
│ │ ├── columnar_upsert.sql
│ │ ├── columnar_vacuum.sql
│ │ ├── columnar_vacuum_udf.sql
│ │ ├── columnar_vectorization.sql
│ │ └── columnar_zstd.sql
└── vendor
│ ├── README.md
│ └── safestringlib
│ ├── .gitattributes
│ ├── .gitignore
│ ├── CMakeLists.txt
│ ├── CODE_OF_CONDUCT.md
│ ├── LICENSE
│ ├── LICENSE©ING.txt
│ ├── README.md
│ ├── include
│ ├── safe_lib.h
│ ├── safe_lib_errno.h
│ ├── safe_lib_errno.h.in
│ ├── safe_mem_lib.h
│ ├── safe_str_lib.h
│ ├── safe_types.h
│ ├── safe_types.h.in
│ └── snprintf_s.h
│ ├── makefile
│ └── safeclib
│ ├── abort_handler_s.c
│ ├── ignore_handler_s.c
│ ├── mem_primitives_lib.c
│ ├── mem_primitives_lib.h
│ ├── memcmp16_s.c
│ ├── memcmp32_s.c
│ ├── memcmp_s.c
│ ├── memcpy16_s.c
│ ├── memcpy32_s.c
│ ├── memcpy_s.c
│ ├── memmove16_s.c
│ ├── memmove32_s.c
│ ├── memmove_s.c
│ ├── memset16_s.c
│ ├── memset32_s.c
│ ├── memset_s.c
│ ├── memzero16_s.c
│ ├── memzero32_s.c
│ ├── memzero_s.c
│ ├── safe_mem_constraint.c
│ ├── safe_mem_constraint.h
│ ├── safe_str_constraint.c
│ ├── safe_str_constraint.h
│ ├── safeclib_private.h
│ ├── snprintf_support.c
│ ├── stpcpy_s.c
│ ├── stpncpy_s.c
│ ├── strcasecmp_s.c
│ ├── strcasestr_s.c
│ ├── strcat_s.c
│ ├── strcmp_s.c
│ ├── strcmpfld_s.c
│ ├── strcpy_s.c
│ ├── strcpyfld_s.c
│ ├── strcpyfldin_s.c
│ ├── strcpyfldout_s.c
│ ├── strcspn_s.c
│ ├── strfirstchar_s.c
│ ├── strfirstdiff_s.c
│ ├── strfirstsame_s.c
│ ├── strisalphanumeric_s.c
│ ├── strisascii_s.c
│ ├── strisdigit_s.c
│ ├── strishex_s.c
│ ├── strislowercase_s.c
│ ├── strismixedcase_s.c
│ ├── strispassword_s.c
│ ├── strisuppercase_s.c
│ ├── strlastchar_s.c
│ ├── strlastdiff_s.c
│ ├── strlastsame_s.c
│ ├── strljustify_s.c
│ ├── strncat_s.c
│ ├── strncpy_s.c
│ ├── strnlen_s.c
│ ├── strnterminate_s.c
│ ├── strpbrk_s.c
│ ├── strprefix_s.c
│ ├── strremovews_s.c
│ ├── strspn_s.c
│ ├── strstr_s.c
│ ├── strtok_s.c
│ ├── strtolowercase_s.c
│ ├── strtouppercase_s.c
│ ├── strzero_s.c
│ ├── wcpcpy_s.c
│ ├── wcscat_s.c
│ ├── wcscpy_s.c
│ ├── wcsncat_s.c
│ ├── wcsncpy_s.c
│ ├── wcsnlen_s.c
│ ├── wmemcmp_s.c
│ ├── wmemcpy_s.c
│ ├── wmemmove_s.c
│ └── wmemset_s.c
├── docker-bake.hcl
├── docker-compose.yml
├── files
├── postgres
│ ├── docker-entrypoint-initdb.d
│ │ └── columnar.sh
│ └── postgresql.conf
└── spilo
│ └── postgres-appliance
│ └── scripts
│ ├── configure_spilo.py
│ ├── multicorn
│ └── after-create.sql
│ ├── mysql_fdw
│ └── after-create.sql
│ ├── parquet_s3_fdw
│ └── after-create.sql
│ ├── post_init.sh
│ └── spilo_commons.py
└── third-party
└── pgxman
├── pgxman_13.yaml
├── pgxman_13_spilo.yaml
├── pgxman_14.yaml
├── pgxman_14_spilo.yaml
├── pgxman_15.yaml
└── pgxman_16.yaml
/.env.example:
--------------------------------------------------------------------------------
1 | POSTGRES_PORT=5432
2 | POSTGRES_USER=postgres
3 | POSTGRES_PASSWORD=hydra
4 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/1.general_bug_report.yaml:
--------------------------------------------------------------------------------
1 | name: Bug Report
2 | description: File a bug report
3 | title: "[Bug]: "
4 | labels: ["bug", "triage"]
5 | body:
6 | - type: markdown
7 | attributes:
8 | value: |
9 | Thanks for taking the time to fill out this bug report! Please ensure you've read our [documentation](https://docs.hydra.so/). If you're unsure as to whether or not you're experiencing a bug, please reach out on our [Discord](https://hydra.so/discord) to gain a better understanding of what you're facing.
10 | - type: textarea
11 | attributes:
12 | label: "What's wrong?"
13 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/config.yml:
--------------------------------------------------------------------------------
1 | blank_issues_enabled: false
2 | contact_links:
3 | - name: 📖 Read the documentation
4 | url: https://docs.hydra.so/
5 | about: Please consult the documentation before opening any issues!
6 | - name: 🙋 Ask a question on Discord
7 | url: https://hydra.so/discord
8 | about: Ask questions about how to use Hydra on our Discord.
9 | - name: 📝 Open a documentation issue or PR
10 | url: https://github.com/hydradatabase/docs
11 | about: You can open issues or PRs directly against our documentation.
12 |
--------------------------------------------------------------------------------
/.github/PULL_REQUEST_TEMPLATE.md:
--------------------------------------------------------------------------------
1 | <!--
2 | Thanks for opening a pull request to Hydra! We've got a few requests to help us review contributions:
3 |
4 | - Make sure that your title neatly summarizes the proposed changes.
5 | - Provide a short overview of the change and the value it adds.
6 | - Share an example to help us understand the change in user experience.
7 | - Please make sure your code changes are covered with tests.
8 | - If your change affects functionality, please include an entry for your PR in CHANGELOG.md
9 | - In the case of new features or big changes, please open a docs PR for the feature at our docs repo:
10 | https://github.com/hydradatabase/docs
11 |
12 | Feel free to ping committers for the review!
13 | -->
14 |
15 | <!-- Include an overview here -->
16 |
17 | ### What's changed?
18 |
19 | <!--
20 | Describe in detail what you've changed.
21 |
22 | A code blurb is best. Changes to features should include an example that is executable by a new user.
23 | If changing documentation, a link to a preview of the page is great.
24 | -->
25 |
--------------------------------------------------------------------------------
/.github/workflows/benchmark.yaml:
--------------------------------------------------------------------------------
1 | name: Benchmark
2 |
3 | on:
4 | workflow_call:
5 | inputs:
6 | repo:
7 | required: true
8 | type: string
9 | default: ghcr.io/hydradatabase/hydra
10 | benchmark:
11 | required: true
12 | type: string
13 | tag:
14 | required: true
15 | type: string
16 |
17 | workflow_dispatch:
18 | inputs:
19 | repo:
20 | required: true
21 | type: choice
22 | default: ghcr.io/hydradatabase/hydra
23 | options:
24 | - ghcr.io/hydradatabase/hydra
25 | - 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo
26 | benchmark:
27 | required: true
28 | type: choice
29 | default: clickbench-1M
30 | options:
31 | - clickbench-10k
32 | - clickbench-1M
33 | - clickbench-100M
34 | - warehouse-1G
35 | - warehouse-10G
36 | tag:
37 | type: string
38 | env:
39 | BENCHMARK: ${{ inputs.benchmark || 'clickbench-1M' }}
40 | REPO: ${{ inputs.repo || 'ghcr.io/hydradatabase/hydra' }}
41 | TAG: ${{ inputs.tag || format('15-{0}', github.sha) }}
42 | BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
43 | BENCHER_TESTBED: gh-4core
44 |
45 | jobs:
46 | benchmarks:
47 |
48 | name: Run Benchmark
49 | # 16gb ram, 4vcpu, 150gb disk
50 | runs-on: benchmarks-ubuntu-latest-4core
51 |
52 | steps:
53 | - uses: actions/setup-node@v3
54 | with:
55 | node-version: 18
56 |
57 | - uses: docker/setup-buildx-action@v2
58 |
59 | - uses: unfor19/install-aws-cli-action@v1
60 | with:
61 | arch: amd64
62 |
63 | - uses: bencherdev/bencher@main
64 |
65 | - name: Checkout benchmarks
66 | uses: actions/checkout@v3
67 | with:
68 | repository: hydradatabase/benchmarks
69 |
70 | - name: Configure AWS credentials
71 | uses: aws-actions/configure-aws-credentials@v1
72 | with:
73 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY || secrets.BENCHMARKS_AWS_ACCESS_KEY_ID }}
74 | aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY || secrets.BENCHMARKS_AWS_SECRET_ACCESS_KEY }}
75 | aws-region: us-east-1
76 | mask-aws-account-id: no
77 |
78 | - name: Login to Amazon ECR
79 | if: ${{ github.repository == 'hydradatabase/hydra-internal' && env.REPO == '011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo' }}
80 | id: spilo-ecr
81 | uses: aws-actions/amazon-ecr-login@v1
82 | with:
83 | registries: "011789831835"
84 |
85 | - name: Run benchmark
86 | run: ./run-gha.sh
87 |
--------------------------------------------------------------------------------
/.github/workflows/build_spilo.yaml:
--------------------------------------------------------------------------------
1 | name: Build Spilo
2 |
3 | on:
4 | workflow_call:
5 | inputs:
6 | production:
7 | required: true
8 | type: boolean
9 | default: false
10 | spilo_repo:
11 | required: true
12 | type: string
13 | workflow_dispatch:
14 | inputs:
15 | production:
16 | required: true
17 | type: boolean
18 | default: false
19 | spilo_repo:
20 | required: true
21 | type: string
22 | default: 011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo
23 |
24 | jobs:
25 | push_spilo:
26 | if: github.repository == 'hydradatabase/hydra-internal'
27 | name: Push Spilo
28 | runs-on: ubuntu-latest
29 | timeout-minutes: 15
30 | steps:
31 | - name: Checkout
32 | uses: actions/checkout@v3
33 | - name: Set up Depot
34 | uses: depot/setup-action@v1
35 | - name: Configure AWS credentials
36 | uses: aws-actions/configure-aws-credentials@v1
37 | with:
38 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
39 | aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
40 | aws-region: us-east-1
41 | mask-aws-account-id: no
42 | - name: Login to Amazon ECR
43 | id: spilo-ecr
44 | uses: aws-actions/amazon-ecr-login@v1
45 | with:
46 | registries: "011789831835"
47 | - name: Bake and push spilo
48 | uses: depot/bake-action@v1
49 | with:
50 | token: ${{ secrets.DEPOT_TOKEN }}
51 | project: ${{ secrets.DEPOT_PROJECT }}
52 | push: true
53 | targets: spilo
54 | set: |
55 | spilo.tags=${{ inputs.spilo_repo }}:${{ inputs.production && github.sha || format('dev-{0}', github.sha) }}
56 | ${{ inputs.production && format('spilo.tags={0}:latest', inputs.spilo_repo) || '' }}
57 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | /.env
3 | /volumes
4 | /tmp
5 |
--------------------------------------------------------------------------------
/.golangci_lint_version:
--------------------------------------------------------------------------------
1 | v1.54.2
2 |
--------------------------------------------------------------------------------
/.images/FasterthanPG.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hydradatabase/columnar/d1f7657ee82265b2b9f1c06fda929bb5b942411b/.images/FasterthanPG.png
--------------------------------------------------------------------------------
/.images/ReadmeBenchmarks.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hydradatabase/columnar/d1f7657ee82265b2b9f1c06fda929bb5b942411b/.images/ReadmeBenchmarks.png
--------------------------------------------------------------------------------
/.images/header.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hydradatabase/columnar/d1f7657ee82265b2b9f1c06fda929bb5b942411b/.images/header.png
--------------------------------------------------------------------------------
/DEVELOPERS.md:
--------------------------------------------------------------------------------
1 | # Developing Hydra
2 |
3 | Most of the work on Hydra exists outside of this repo through various extensions that we add to the image.
4 | The hydra project exists for coordination and to build the final Docker image which contains Postgres and
5 | these extensions.
6 |
7 | Currently active projects are:
8 |
9 | * [the Hydra columnar engine](https://github.com/hydradatabase/citus)
10 |
11 | ## Build
12 |
13 | The Hydra Docker image is based on [zalando/spilo](https://github.com/zalando/spilo).
14 | The image is a distribution of Spilo and the [Columnar PostgreSQL extension](https://github.com/hydradatabase/citus).
15 |
16 | To build, run:
17 |
18 | ```
19 | TAG=1234 TARGET=hydra make docker_build
20 | ```
21 |
22 | ## Image Build Tags
23 |
24 | Image build tag is in the format of `${SPILO_SHA}_${COLUMNAR_EXT_SHA}`, e.g. `72fb97e_ff32dd9`.
25 | The `latest` tag is always tagged to the latest main branch.
26 |
27 | ## Hydra Production Version
28 |
29 | The `HYDRA_PROD_VER` file stores the version that Hydra runs in production.
30 | Please bump it when a new production version is released.
31 |
32 | ## Spilo Version Update
33 |
34 | Hydra Docker build overrides the following Spilo scripts to enable extra PostgreSQL extensions:
35 |
36 | * [configure_spilo.py](https://github.com/zalando/spilo/blob/master/postgres-appliance/scripts/configure_spilo.py)
37 | * [spilo_commons.py](https://github.com/zalando/spilo/blob/master/postgres-appliance/scripts/spilo_commons.py)
38 | * [post_init.sh](https://github.com/zalando/spilo/blob/master/postgres-appliance/scripts/post_init.sh)
39 |
40 | To update to a newer Spilo version, you need to copy the above files from the specific Spilo version and add the corresponding bits to enable the Columnar extensions.
41 | Make sure you use `diff` to understand what needs to be added first before updating.
42 | In the future, we may provide a script to simplify the update process.
43 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | #syntax=docker/dockerfile:1
2 |
3 | FROM postgres_base
4 |
5 | RUN set -eux; \
6 | apt-get update; \
7 | apt-get install -y --no-install-recommends \
8 | curl \
9 | ca-certificates \
10 | ; \
11 | rm -rf /var/lib/apt/lists/*
12 |
13 | # columnar ext
14 | # NOTE(owenthereal): build columnar with pgxman in this repo
15 | COPY --from=columnar /pg_ext /
16 |
17 | COPY files/postgres/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d/
18 |
19 | ARG POSTGRES_BASE_VERSION
20 | # Install pgxman extensions
21 | # Always force rebuild of this layer
22 | ARG TIMESTAMP=1
23 | COPY third-party/pgxman /tmp/pgxman/
24 | RUN curl -sfL https://install.pgx.sh | sh -s -- /tmp/pgxman/pgxman_${POSTGRES_BASE_VERSION}.yaml && \
25 | pgxman install pgsql-http=1.6.0 --pg ${POSTGRES_BASE_VERSION} --yes && \
26 | rm -rf /tmp/pgxman
27 |
--------------------------------------------------------------------------------
/Dockerfile.spilo:
--------------------------------------------------------------------------------
1 | #syntax=docker/dockerfile:1
2 |
3 | FROM spilo_base
4 |
5 | RUN set -eux; \
6 | apt-get update; \
7 | apt-get install -y --no-install-recommends \
8 | curl \
9 | ca-certificates \
10 | ; \
11 | rm -rf /var/lib/apt/lists/*
12 |
13 | # See https://github.com/docker-library/postgres/issues/415
14 | RUN set -eux; \
15 | locale-gen en_US.UTF-8; \
16 | echo "LC_ALL=en_US.UTF-8" >> /etc/environment; \
17 | echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen; \
18 | echo "LANG=en_US.UTF-8" > /etc/locale.conf
19 |
20 | # columnar ext
21 | # NOTE(owenthereal): build columnar with pgxman in this repo
22 | COPY --from=columnar_13 /pg_ext /
23 | COPY --from=columnar_14 /pg_ext /
24 | COPY --from=columnar_15 /pg_ext /
25 |
26 | # configuration
27 | COPY files/spilo/postgres-appliance/scripts /scripts/
28 |
29 | ARG POSTGRES_BASE_VERSION
30 | # Default envs
31 | ENV PGVERSION=${POSTGRES_BASE_VERSION} SPILO_PROVIDER=local PGUSER_SUPERUSER=postgres PGPASSWORD_SUPERUSER=hydra
32 |
33 | # Install pgxman extensions
34 | # Always force rebuild of this layer
35 | ARG TIMESTAMP=1
36 | COPY third-party/pgxman /tmp/pgxman/
37 | RUN curl -sfL https://install.pgx.sh | sh -s -- /tmp/pgxman/pgxman_13_spilo.yaml /tmp/pgxman/pgxman_14_spilo.yaml && \
38 | rm -rf /tmp/pgxman
39 |
--------------------------------------------------------------------------------
/HYDRA_PROD_VER:
--------------------------------------------------------------------------------
1 | 3cdf8367ae8717e67c60daf4e5eafd7134de589a
2 |
--------------------------------------------------------------------------------
/VERSION:
--------------------------------------------------------------------------------
1 | 1.1.2
2 |
--------------------------------------------------------------------------------
/acceptance/.golangci.yml:
--------------------------------------------------------------------------------
1 | linters:
2 | presets:
3 | - bugs
4 | - comment
5 | - error
6 | - import
7 | - sql
8 | - test
9 | - unused
10 | disable:
11 | # unwanted
12 | - depguard
13 | - exhaustivestruct
14 | - exhaustruct
15 | - gosec
16 | - paralleltest
17 | # deprecated
18 | - deadcode
19 | - scopelint
20 | - varcheck
21 | # issues with generics
22 | - rowserrcheck
23 | - sqlclosecheck
24 | - structcheck
25 |
--------------------------------------------------------------------------------
/acceptance/fixtures/mysql.sql:
--------------------------------------------------------------------------------
1 | create database test;
2 | use test;
3 |
4 | CREATE TABLE warehouse (
5 | warehouse_id int primary key,
6 | warehouse_name varchar(255),
7 | warehouse_created timestamp
8 | );
9 |
--------------------------------------------------------------------------------
/acceptance/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/hydradatabase/hydra/acceptance
2 |
3 | go 1.21
4 |
5 | require (
6 | github.com/google/uuid v1.3.0
7 | github.com/jackc/pgx/v5 v5.0.4
8 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd
9 | github.com/rs/xid v1.2.1
10 | )
11 |
12 | require (
13 | github.com/jackc/pgpassfile v1.0.0 // indirect
14 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
15 | github.com/jackc/puddle/v2 v2.0.0 // indirect
16 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
17 | golang.org/x/text v0.3.8 // indirect
18 | )
19 |
--------------------------------------------------------------------------------
/acceptance/go.sum:
--------------------------------------------------------------------------------
1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
5 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
6 | github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
7 | github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
8 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
9 | github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
10 | github.com/jackc/pgx/v5 v5.0.4 h1:r5O6y84qHX/z/HZV40JBdx2obsHz7/uRj5b+CcYEdeY=
11 | github.com/jackc/pgx/v5 v5.0.4/go.mod h1:U0ynklHtgg43fue9Ly30w3OCSTDPlXjig9ghrNGaguQ=
12 | github.com/jackc/puddle/v2 v2.0.0 h1:Kwk/AlLigcnZsDssc3Zun1dk1tAtQNPaBBxBHWn0Mjc=
13 | github.com/jackc/puddle/v2 v2.0.0/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc=
14 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd h1:nIzoSW6OhhppWLm4yqBwZsKJlAayUu5FGozhrF3ETSM=
15 | github.com/joeshaw/envdecode v0.0.0-20200121155833-099f1fc765bd/go.mod h1:MEQrHur0g8VplbLOv5vXmDzacSaH9Z7XhcgsSh1xciU=
16 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
18 | github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
19 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
20 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
21 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
22 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
23 | github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
24 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
25 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
26 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
27 | golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
28 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
29 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
30 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
31 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
32 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
33 |
--------------------------------------------------------------------------------
/acceptance/shared/pg.go:
--------------------------------------------------------------------------------
1 | package shared
2 |
3 | import (
4 | "context"
5 | "regexp"
6 | "testing"
7 |
8 | "github.com/jackc/pgx/v5/pgxpool"
9 | )
10 |
11 | var (
12 | regexpPGVersion = regexp.MustCompile(`^PostgreSQL (\d+)`)
13 | )
14 |
15 | func QueryPGVersion(t *testing.T, ctx context.Context, pool *pgxpool.Pool) PGVersion {
16 | row := pool.QueryRow(ctx, "SELECT VERSION();")
17 |
18 | var version string
19 | if err := row.Scan(&version); err != nil {
20 | t.Fatal(err)
21 | }
22 |
23 | matches := regexpPGVersion.FindStringSubmatch(version)
24 | if len(matches) == 0 {
25 | t.Fatalf("failed to parse pg_config --version output: %s", version)
26 | }
27 |
28 | return PGVersion(matches[1])
29 | }
30 |
--------------------------------------------------------------------------------
/bin/autolink_changelog.rb:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env ruby
2 |
3 | # usage: bin/autolink_changelog.rb < CHANGELOG.rb > LINKED-CHANGELOG.rb
4 |
5 | require 'set'
6 |
7 | PR_REGEXP = /(?<!\[)(#\d+)/
8 | COMMIT_REGEXP = /(?<!\[)\b([\da-f]{7,40})\b/
9 | LINK_REGEXP = /^\[([^\]]+)\]:\s[^\n]+/
10 |
11 | prs = Set.new
12 | commits = Set.new
13 | links = Set.new
14 |
15 | while line = STDIN.gets
16 | if m = line.match(LINK_REGEXP)
17 | links << m[1]
18 | else
19 | line = line.gsub(PR_REGEXP) do |match|
20 | prs << match
21 | "[#{match}][]"
22 | end
23 | line = line.gsub(COMMIT_REGEXP) do |match|
24 | commits << match
25 | "[#{match}][]"
26 | end
27 | end
28 | puts line
29 | end
30 |
31 | prs.each do |pr|
32 | next if links.include?(pr)
33 | puts "[#{pr}]: https://github.com/hydradatabase/hydra/pull/#{pr.sub('#', '')}"
34 | end
35 | commits.each do |commit|
36 | next if links.include?(commit)
37 | puts "[#{commit}]: https://github.com/hydradatabase/hydra/commit/#{commit}"
38 | end
39 |
--------------------------------------------------------------------------------
/columnar/.gitattributes:
--------------------------------------------------------------------------------
1 | * whitespace=space-before-tab,trailing-space
2 | *.[chly] whitespace=space-before-tab,trailing-space,indent-with-non-tab,tabwidth=4
3 | *.dsl whitespace=space-before-tab,trailing-space,tab-in-indent
4 | *.patch -whitespace
5 | *.pl whitespace=space-before-tab,trailing-space,tabwidth=4
6 | *.po whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eof
7 | *.sgml whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eol
8 | *.x[ms]l whitespace=space-before-tab,trailing-space,tab-in-indent
9 |
10 | # Avoid confusing ASCII underlines with leftover merge conflict markers
11 | README conflict-marker-size=32
12 | README.* conflict-marker-size=32
13 |
14 | # Certain data files that contain special whitespace, and other special cases
15 | *.data -whitespace
16 |
17 | # Test output files that contain extra whitespace
18 | *.out -whitespace
19 | src/test/regress/output/*.source -whitespace
20 |
21 | # These files are maintained or generated elsewhere. We take them as is.
22 | configure -whitespace
23 |
24 | # all C files (implementation and header) use our style...
25 | *.[ch] citus-style
26 |
27 | # except these exceptions...
28 | src/backend/distributed/utils/citus_outfuncs.c -citus-style
29 | src/backend/distributed/utils/pg11_snprintf.c -citus-style
30 | src/backend/distributed/deparser/ruleutils_11.c -citus-style
31 | src/backend/distributed/deparser/ruleutils_12.c -citus-style
32 | src/backend/distributed/deparser/ruleutils_13.c -citus-style
33 | src/backend/distributed/deparser/ruleutils_14.c -citus-style
34 | src/backend/distributed/commands/index_pg_source.c -citus-style
35 |
36 | src/include/distributed/citus_nodes.h -citus-style
37 | /vendor/** -citus-style
38 |
39 | # Hide diff on github by default for copied udfs
40 | src/backend/distributed/sql/udfs/*/[123456789]*.sql linguist-generated=true
41 |
--------------------------------------------------------------------------------
/columnar/.gitignore:
--------------------------------------------------------------------------------
1 | # Global excludes across all subdirectories
2 | *.o
3 | *.so
4 | *.so.[0-9]
5 | *.so.[0-9].[0-9]
6 | *.sl
7 | *.sl.[0-9]
8 | *.sl.[0-9].[0-9]
9 | *.dylib
10 | *.dll
11 | *.a
12 | *.mo
13 | *.pot
14 | objfiles.txt
15 | .deps/
16 | *.gcno
17 | *.gcda
18 | *.gcov
19 | *.gcov.out
20 | lcov.info
21 | coverage/
22 | *.vcproj
23 | *.vcxproj
24 | win32ver.rc
25 | *.exe
26 | lib*dll.def
27 | lib*.pc
28 | *.bc
29 |
30 | # Local excludes in root directory
31 | /config.log
32 | /config.status
33 | /pgsql.sln
34 | /pgsql.sln.cache
35 | /Debug/
36 | /Release/
37 | /autom4te.cache
38 | /Makefile.global
39 | /src/Makefile.custom
40 | /compile_commands.json
41 |
42 | # temporary files vim creates
43 | *.swp
44 |
45 | # vscode
46 | .vscode/*
47 | .ccls
48 |
49 | # output from diff normalization that shouldn't be commited
50 | *.unmodified
51 | *.modified
52 |
53 | # style related temporary outputs
54 | *.uncrustify
55 |
--------------------------------------------------------------------------------
/columnar/.ignore:
--------------------------------------------------------------------------------
1 | /vendor
2 |
--------------------------------------------------------------------------------
/columnar/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM ubuntu:22.04 as setup
2 |
3 | ARG POSTGRES_BASE_VERSION=13
4 |
5 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install gnupg postgresql-common -y
6 | RUN sh /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
7 | RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get upgrade -y && apt-get install lsb-release gcc make libssl-dev autoconf pkg-config postgresql-${POSTGRES_BASE_VERSION} postgresql-server-dev-${POSTGRES_BASE_VERSION} libcurl4-gnutls-dev liblz4-dev libzstd-dev -y
8 |
9 | FROM setup as builder
10 |
11 | RUN --mount=target=/columnar,rw cd /columnar && ./configure && cd src/backend/columnar && DESTDIR=/pg_ext make install
12 |
13 | FROM setup as checker
14 |
15 | WORKDIR /columnar
16 | COPY . .
17 |
18 | RUN chown -R postgres:postgres .
19 |
20 | USER postgres
21 | ENV PATH=/usr/lib/postgresql/${POSTGRES_BASE_VERSION}/bin:$PATH
22 | RUN ./configure && make clean extension
23 |
24 | USER root
25 | RUN make install-all
26 |
27 | USER postgres
28 | RUN make check-all
29 |
30 | USER root
31 | RUN DESTDIR=/pg_ext make -C /columnar/src/backend/columnar install
32 |
33 | FROM scratch as output
34 |
35 | COPY --from=builder /pg_ext /pg_ext
36 |
--------------------------------------------------------------------------------
/columnar/Makefile:
--------------------------------------------------------------------------------
1 | # Citus toplevel Makefile
2 |
3 | # Hint that configure should be run first
4 | ifeq (,$(wildcard Makefile.global))
5 | $(error ./configure needs to be run before compiling Citus)
6 | endif
7 |
8 | include Makefile.global
9 |
10 | citus_subdir = .
11 | citus_top_builddir = .
12 | extension_dir = $(shell $(PG_CONFIG) --sharedir)/extension
13 |
14 | all: extension
15 |
16 | # build extension
17 | extension: $(citus_top_builddir)/src/include/citus_version.h
18 | $(MAKE) -C src/backend/columnar all
19 |
20 | install-extension: extension
21 | $(MAKE) -C src/backend/columnar install
22 |
23 | install-headers: extension
24 | @mkdir -p '$(DESTDIR)$(includedir_server)'
25 | $(INSTALL_DATA) $(citus_top_builddir)/src/include/citus_version.h '$(DESTDIR)$(includedir_server)/'
26 |
27 | clean-extension:
28 | $(MAKE) -C src/backend/columnar/ clean
29 |
30 | .PHONY: extension install-extension clean-extension
31 |
32 | # Add to generic targets
33 | install: install-extension install-headers
34 |
35 | install-all: install-headers
36 | $(MAKE) -C src/backend/columnar/ install-all
37 |
38 | clean: clean-extension clean-regression
39 |
40 | # apply or check style
41 | reindent:
42 | ${citus_abs_top_srcdir}/ci/fix_style.sh
43 |
44 | .PHONY: reindent
45 |
46 | check: all install-all check-all
47 |
48 | check-all:
49 | $(MAKE) -C src/test/regress check-all
50 |
51 | clean-regression:
52 | $(MAKE) -C src/test/regress clean-regression
53 |
54 | .PHONY: all check clean install install-all
55 |
--------------------------------------------------------------------------------
/columnar/README.md:
--------------------------------------------------------------------------------
1 | # Columnar
2 |
3 | ## Documentation
4 |
5 | Please see:
6 |
7 | * [Using Hydra Columnar](https://docs.hydra.so/concepts/using-hydra-columnar)
8 | * [Updates and Deletes](https://docs.hydra.so/concepts/updates-and-deletes)
9 | * [Row vs Column Tables](https://docs.hydra.so/organize/data-modeling/row-vs-column-tables)
10 |
11 | For more depth, you can also read our release blog posts:
12 |
13 | * [Performance improvements](https://blog.hydra.so/blog/2022-12-13-how-we-built-fastest-postgres-db-for-analytics) (2022-12-13)
14 | * [Updates and deletes](https://blog.hydra.so/blog/2023-03-02-columnar-updates-and-deletes) (2023-03-02)
15 |
16 | Hydra Columnar is a fork from [Citus Columnar](https://github.com/citusdata/citus/tree/main/src/backend/columnar) c. April, 2022.
17 |
--------------------------------------------------------------------------------
/columnar/aclocal.m4:
--------------------------------------------------------------------------------
1 | dnl aclocal.m4
2 | m4_include([config/general.m4])
3 |
--------------------------------------------------------------------------------
/columnar/autogen.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | #
3 | # autogen.sh converts configure.in to configure and creates
4 | # citus_config.h.in. The resuting resulting files are checked into
5 | # the SCM, to avoid everyone needing autoconf installed.
6 |
7 | autoreconf -f
8 |
--------------------------------------------------------------------------------
/columnar/cgmanifest.json:
--------------------------------------------------------------------------------
1 | {
2 | "Registrations": [
3 | {
4 | "Component": {
5 | "Type": "git",
6 | "git": {
7 | "RepositoryUrl": "https://github.com/intel/safestringlib",
8 | "CommitHash": "245c4b8cff1d2e7338b7f3a82828fc8e72b29549"
9 | }
10 | },
11 | "DevelopmentDependency": false
12 | },
13 | {
14 | "Component": {
15 | "Type": "git",
16 | "git": {
17 | "RepositoryUrl": "https://github.com/postgres/postgres",
18 | "CommitHash": "29be9983a64c011eac0b9ee29895cce71e15ea77"
19 | }
20 | },
21 | "license": "PostgreSQL",
22 | "licenseDetail": [
23 | "Portions Copyright (c) 1996-2010, The PostgreSQL Global Development Group",
24 | "",
25 | "Portions Copyright (c) 1994, The Regents of the University of California",
26 | "",
27 | "Permission to use, copy, modify, and distribute this software and its documentation for ",
28 | "any purpose, without fee, and without a written agreement is hereby granted, provided ",
29 | "that the above copyright notice and this paragraph and the following two paragraphs appear ",
30 | "in all copies.",
31 | "",
32 | "IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, ",
33 | "INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS ",
34 | "SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE ",
35 | "POSSIBILITY OF SUCH DAMAGE.",
36 | "",
37 | "THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, ",
38 | "THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED ",
39 | "HEREUNDER IS ON AN \"AS IS\" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE ",
40 | "MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS."
41 | ],
42 | "version": "0.0.1",
43 | "DevelopmentDependency": false
44 | }
45 |
46 | ]
47 | }
48 |
--------------------------------------------------------------------------------
/columnar/docker-build.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 |
3 | REPO=hydra/columnar
4 | TAG=latest
5 |
6 | docker build -t $REPO:$TAG .
--------------------------------------------------------------------------------
/columnar/prep_buildtree:
--------------------------------------------------------------------------------
1 | #! /bin/sh
2 | #
3 | # Citus copy of PostgreSQL's config/prep_buildtree
4 | #
5 | # This script prepares a Citus build tree for an out-of-tree/VPATH
6 | # build. It is intended to be run by the configure script.
7 |
8 | me=`basename $0`
9 |
10 | help="\
11 | Usage: $me sourcetree [buildtree]"
12 |
13 | if test -z "$1"; then
14 | echo "$help" 1>&2
15 | exit 1
16 | elif test x"$1" = x"--help"; then
17 | echo "$help"
18 | exit 0
19 | fi
20 |
21 | unset CDPATH
22 |
23 | sourcetree=`cd $1 && pwd`
24 |
25 | buildtree=`cd ${2:-'.'} && pwd`
26 |
27 | # We must not auto-create the subdirectories holding built documentation.
28 | # If we did, it would interfere with installation of prebuilt docs from
29 | # the source tree, if a VPATH build is done from a distribution tarball.
30 | # See bug #5595.
31 | for item in `find "$sourcetree" -type d \( \( -name CVS -prune \) -o \( -name .git -prune \) -o -print \) | grep -v "$sourcetree/doc/src/sgml/\+"`; do
32 | subdir=`expr "$item" : "$sourcetree\(.*\)"`
33 | if test ! -d "$buildtree/$subdir"; then
34 | mkdir -p "$buildtree/$subdir" || exit 1
35 | fi
36 | done
37 |
38 | for item in `find "$sourcetree" -not -path '*/.git/hg/*' \( -name Makefile -print -o -name GNUmakefile -print \)`; do
39 | filename=`expr "$item" : "$sourcetree\(.*\)"`
40 | if test ! -f "${item}.in"; then
41 | if cmp "$item" "$buildtree/$filename" >/dev/null 2>&1; then : ; else
42 | ln -fs "$item" "$buildtree/$filename" || exit 1
43 | fi
44 | fi
45 | done
46 |
47 | exit 0
48 |
--------------------------------------------------------------------------------
/columnar/src/backend/.gitignore:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hydradatabase/columnar/d1f7657ee82265b2b9f1c06fda929bb5b942411b/columnar/src/backend/.gitignore
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/.gitattributes:
--------------------------------------------------------------------------------
1 | * whitespace=space-before-tab,trailing-space
2 | *.[chly] whitespace=space-before-tab,trailing-space,indent-with-non-tab,tabwidth=4
3 | *.dsl whitespace=space-before-tab,trailing-space,tab-in-indent
4 | *.patch -whitespace
5 | *.pl whitespace=space-before-tab,trailing-space,tabwidth=4
6 | *.po whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eof
7 | *.sgml whitespace=space-before-tab,trailing-space,tab-in-indent,-blank-at-eol
8 | *.x[ms]l whitespace=space-before-tab,trailing-space,tab-in-indent
9 |
10 | # Avoid confusing ASCII underlines with leftover merge conflict markers
11 | README conflict-marker-size=32
12 | README.* conflict-marker-size=32
13 |
14 | # Certain data files that contain special whitespace, and other special cases
15 | *.data -whitespace
16 |
17 | # Test output files that contain extra whitespace
18 | *.out -whitespace
19 | src/test/regress/output/*.source -whitespace
20 |
21 | # These files are maintained or generated elsewhere. We take them as is.
22 | configure -whitespace
23 |
24 | # all C files (implementation and header) use our style...
25 | *.[ch] citus-style
26 |
27 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/.gitignore:
--------------------------------------------------------------------------------
1 | /build/
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/Makefile:
--------------------------------------------------------------------------------
1 | citus_subdir = src/backend/columnar
2 | citus_top_builddir = ../../..
3 | safestringlib_srcdir = $(citus_abs_top_srcdir)/vendor/safestringlib
4 | SUBDIRS = . safeclib vectorization vectorization/types vectorization/nodes
5 | SUBDIRS +=
6 | ENSURE_SUBDIRS_EXIST := $(shell mkdir -p $(SUBDIRS))
7 | OBJS += \
8 | $(patsubst $(citus_abs_srcdir)/%.c,%.o,$(foreach dir,$(SUBDIRS), $(sort $(wildcard $(citus_abs_srcdir)/$(dir)/*.c))))
9 |
10 | MODULE_big = columnar
11 | PG_CPPFLAGS += -I$(libpq_srcdir) -I$(safestringlib_srcdir)/include
12 |
13 | ifdef COLUMNAR_O3
14 | PG_CFLAGS += -O3
15 | endif
16 |
17 | EXTENSION = columnar
18 |
19 | template_sql_files = $(patsubst $(citus_abs_srcdir)/%,%,$(wildcard $(citus_abs_srcdir)/sql/*.sql))
20 | template_downgrade_sql_files = $(patsubst $(citus_abs_srcdir)/sql/downgrades/%,%,$(wildcard $(citus_abs_srcdir)/sql/downgrades/*.sql))
21 | generated_sql_files = $(patsubst %,$(citus_abs_srcdir)/build/%,$(template_sql_files))
22 | generated_downgrade_sql_files += $(patsubst %,$(citus_abs_srcdir)/build/sql/%,$(template_downgrade_sql_files))
23 | # All citus--*.sql files that are used to upgrade between versions
24 | DATA_built = $(generated_sql_files)
25 |
26 | include $(citus_top_builddir)/Makefile.global
27 |
28 | SQL_DEPDIR=.deps/sql
29 | SQL_BUILDDIR=build/sql
30 |
31 | $(generated_sql_files): $(citus_abs_srcdir)/build/%: %
32 | @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR)
33 | @# -MF is used to store dependency files(.Po) in another directory for separation
34 | @# -MT is used to change the target of the rule emitted by dependency generation.
35 | @# -P is used to inhibit generation of linemarkers in the output from the preprocessor.
36 | @# -undef is used to not predefine any system-specific or GCC-specific macros.
37 | @# `man cpp` for further information
38 | cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ lt; > $@
39 |
40 | $(generated_downgrade_sql_files): $(citus_abs_srcdir)/build/sql/%: sql/downgrades/%
41 | @mkdir -p $(citus_abs_srcdir)/$(SQL_DEPDIR) $(citus_abs_srcdir)/$(SQL_BUILDDIR)
42 | @# -MF is used to store dependency files(.Po) in another directory for separation
43 | @# -MT is used to change the target of the rule emitted by dependency generation.
44 | @# -P is used to inhibit generation of linemarkers in the output from the preprocessor.
45 | @# -undef is used to not predefine any system-specific or GCC-specific macros.
46 | @# `man cpp` for further information
47 | cd $(citus_abs_srcdir) && cpp -undef -w -P -MMD -MP -MF$(SQL_DEPDIR)/$(*F).Po -MT$@ lt; > $@
48 |
49 | SQL_Po_files := $(wildcard $(SQL_DEPDIR)/*.Po)
50 | ifneq (,$(SQL_Po_files))
51 | include $(SQL_Po_files)
52 | endif
53 |
54 | .PHONY: install-all
55 | install-all: install
56 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/columnar.control:
--------------------------------------------------------------------------------
1 | comment = 'Hydra Columnar extension'
2 | default_version = '11.1-12'
3 | module_pathname = '$libdir/columnar'
4 | relocatable = false
5 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/mod.c:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * mod.c
4 | *
5 | * This file contains module-level definitions.
6 | *
7 | * Copyright (c) 2016, Citus Data, Inc.
8 | *
9 | * $Id$
10 | *
11 | *-------------------------------------------------------------------------
12 | */
13 |
14 | #include "postgres.h"
15 |
16 | #include "fmgr.h"
17 |
18 | #include "citus_version.h"
19 |
20 | #include "columnar/columnar.h"
21 | #include "columnar/columnar_tableam.h"
22 |
23 |
24 | PG_MODULE_MAGIC;
25 |
26 | void _PG_init(void);
27 |
28 | void
29 | _PG_init(void)
30 | {
31 | columnar_init();
32 | }
33 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/safeclib/abort_handler_s.c:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * abort_handler_s.c
3 | *
4 | * 2012, Jonathan Toppins <jtoppins@users.sourceforge.net>
5 | *
6 | * Copyright (c) 2012 Cisco Systems
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #include "safeclib_private.h"
33 |
34 | /**
35 | * NAME
36 | * abort_handler_s
37 | *
38 | * SYNOPSIS
39 | * #include "safe_lib.h"
40 | * void abort_handler_s(const char *msg, void *ptr, errno_t error)
41 | *
42 | * DESCRIPTION
43 | * This function writes a message on the standard error stream in
44 | * an implementation-defined format. The message shall include the
45 | * string pointed to by msg. The abort_handler_s function then calls
46 | * the abort function.
47 | *
48 | * SPECIFIED IN
49 | * ISO/IEC JTC1 SC22 WG14 N1172, Programming languages, environments
50 | * and system software interfaces, Extensions to the C Library,
51 | * Part I: Bounds-checking interfaces
52 | *
53 | * INPUT PARAMETERS
54 | * msg Pointer to the message describing the error
55 | *
56 | * ptr Pointer to aassociated data. Can be NULL.
57 | *
58 | * error The error code encountered.
59 | *
60 | * RETURN VALUE
61 | * Does not return to caller.
62 | *
63 | * ALSO SEE
64 | * ignore_handler_s()
65 | *
66 | */
67 |
68 | void abort_handler_s(const char *msg, void *ptr, errno_t error)
69 | {
70 | slprintf("ABORT CONSTRAINT HANDLER: (%u) %s\n", error,
71 | (msg) ? msg : "Null message");
72 | slabort();
73 | }
74 | EXPORT_SYMBOL(abort_handler_s)
75 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/safeclib/ignore_handler_s.c:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * ignore_handler_s.c
3 | *
4 | * 2012, Jonathan Toppins <jtoppins@users.sourceforge.net>
5 | *
6 | * Copyright (c) 2012 Cisco Systems
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #include "safeclib_private.h"
33 |
34 | /**
35 | * NAME
36 | * ignore_handler_s
37 | *
38 | * SYNOPSIS
39 | * #include "safe_lib.h"
40 | * void ignore_handler_s(const char *msg, void *ptr, errno_t error)
41 | *
42 | * DESCRIPTION
43 | * This function simply returns to the caller.
44 | *
45 | * SPECIFIED IN
46 | * ISO/IEC JTC1 SC22 WG14 N1172, Programming languages, environments
47 | * and system software interfaces, Extensions to the C Library,
48 | * Part I: Bounds-checking interfaces
49 | *
50 | * INPUT PARAMETERS
51 | * msg Pointer to the message describing the error
52 | *
53 | * ptr Pointer to aassociated data. Can be NULL.
54 | *
55 | * error The error code encountered.
56 | *
57 | * RETURN VALUE
58 | * Returns no value.
59 | *
60 | * ALSO SEE
61 | * abort_handler_s()
62 | *
63 | */
64 |
65 | void ignore_handler_s(const char *msg, void *ptr, errno_t error)
66 | {
67 |
68 | sldebug_printf("IGNORE CONSTRAINT HANDLER: (%u) %s\n", error,
69 | (msg) ? msg : "Null message");
70 | return;
71 | }
72 | EXPORT_SYMBOL(ignore_handler_s)
73 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/safeclib/mem_primitives_lib.h:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * mem_primitives_lib.h - Unguarded Memory Copy Routines
3 | *
4 | * October 2008, Bo Berry
5 | *
6 | * Copyright (c) 2008-2011 by Cisco Systems, Inc
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #ifndef __MEM_PRIMITIVES_LIB_H__
33 | #define __MEM_PRIMITIVES_LIB_H__
34 |
35 | #include "safeclib_private.h"
36 |
37 | /*
38 | * These are prototypes for _unguarded_ memory routines. The caller must
39 | * validate all parameters prior to invocation. Useful for diagnostics
40 | * and system initialization processing.
41 | */
42 |
43 | /* moves (handles overlap) memory */
44 | extern void
45 | mem_prim_move(void *dest, const void *src, uint32_t length);
46 |
47 |
48 | /* uint8_t moves (handles overlap) memory */
49 | extern void
50 | mem_prim_move8(uint8_t *dest, const uint8_t *src, uint32_t length);
51 |
52 | /* uint16_t moves (handles overlap) memory */
53 | extern void
54 | mem_prim_move16(uint16_t *dest, const uint16_t *src, uint32_t length);
55 |
56 | /* uint32_t moves (handles overlap) memory */
57 | extern void
58 | mem_prim_move32(uint32_t *dest, const uint32_t *src, uint32_t length);
59 |
60 |
61 | /* set bytes */
62 | extern void
63 | mem_prim_set(void *dest, uint32_t dmax, uint8_t value);
64 |
65 | /* set uint16_ts */
66 | extern void
67 | mem_prim_set16(uint16_t *dest, uint32_t dmax, uint16_t value);
68 |
69 | /* set uint32_ts */
70 | extern void
71 | mem_prim_set32(uint32_t *dest, uint32_t dmax, uint32_t value);
72 |
73 |
74 | #endif /* __MEM_PRIMITIVES_LIB_H__ */
75 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/safeclib/safe_mem_constraint.h:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * safe_mem_constraint.h
3 | *
4 | * October 2008, Bo Berry
5 | *
6 | * Copyright (c) 2008, 2009 by Cisco Systems, Inc.
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #ifndef __SAFE_MEM_CONSTRAINT_H__
33 | #define __SAFE_MEM_CONSTRAINT_H__
34 |
35 | #include "safeclib_private.h"
36 |
37 | /*
38 | * Function used by the libraries to invoke the registered
39 | * runtime-constraint handler. Always needed.
40 | */
41 | extern void invoke_safe_mem_constraint_handler(
42 | const char *msg,
43 | void *ptr,
44 | errno_t error);
45 |
46 | #endif /* __SAFE_MEM_CONSTRAINT_H__ */
47 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/safeclib/safeclib_private.h:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * safeclib_private.h - Internal library references
3 | *
4 | * 2012, Jonathan Toppins <jtoppins@users.sourceforge.net>
5 | *
6 | * Copyright (c) 2012, 2013 by Cisco Systems, Inc
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #ifndef __SAFECLIB_PRIVATE_H__
33 | #define __SAFECLIB_PRIVATE_H__
34 |
35 | #include "citus_config.h"
36 | #ifdef __KERNEL__
37 | /* linux kernel environment */
38 |
39 | #include <linux/kernel.h>
40 | #include <linux/module.h>
41 | #include <linux/ctype.h>
42 |
43 | #define RCNEGATE(x) ( -(x) )
44 |
45 | #define slprintf(...) printk(KERN_EMERG __VA_ARGS__)
46 | #define slabort()
47 | #ifdef DEBUG
48 | #define sldebug_printf(...) printk(KERN_DEBUG __VA_ARGS__)
49 | #endif
50 |
51 | #else /* !__KERNEL__ */
52 |
53 | #if HAVE_CONFIG_H
54 | #include "config.h"
55 | #endif
56 |
57 | #include <stdio.h>
58 | #ifdef STDC_HEADERS
59 | # include <ctype.h>
60 | # include <stdlib.h>
61 | # include <stddef.h>
62 | #else
63 | # ifdef HAVE_STDLIB_H
64 | # include <stdlib.h>
65 | # endif
66 | #endif
67 | #ifdef HAVE_STRING_H
68 | # if !defined STDC_HEADERS && defined HAVE_MEMORY_H
69 | # include <memory.h>
70 | # endif
71 | # include <string.h>
72 | #endif
73 | #ifdef HAVE_LIMITS_H
74 | # include <limits.h>
75 | #endif
76 |
77 | #define EXPORT_SYMBOL(sym)
78 | #define RCNEGATE(x) (x)
79 |
80 | #define slprintf(...) fprintf(stderr, __VA_ARGS__)
81 | #define slabort() abort()
82 | #ifdef DEBUG
83 | #define sldebug_printf(...) printf(__VA_ARGS__)
84 | #endif
85 |
86 | #endif /* __KERNEL__ */
87 |
88 | #ifndef sldebug_printf
89 | #define sldebug_printf(...)
90 | #endif
91 |
92 | #include "safe_lib.h"
93 |
94 | #endif /* __SAFECLIB_PRIVATE_H__ */
95 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--10.0-1--10.0-2.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.0-1--10.0-2.sql
2 |
3 | -- grant read access for columnar metadata tables to unprivileged user
4 | GRANT USAGE ON SCHEMA columnar TO PUBLIC;
5 | GRANT SELECT ON ALL tables IN SCHEMA columnar TO PUBLIC ;
6 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--10.0-2--10.0-3.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.0-2--10.0-3.sql
2 |
3 | -- placeholder, no changes
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--10.0-3--10.1-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.0-3--10.1-1.sql
2 |
3 | -- Drop foreign keys between columnar metadata tables.
4 | -- Postgres assigns different names to those foreign keys in PG11, so act accordingly.
5 | DO $proc$
6 | BEGIN
7 | IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
8 | EXECUTE $
9 | ALTER TABLE columnar.chunk DROP CONSTRAINT chunk_storage_id_stripe_num_chunk_group_num_fkey;
10 | ALTER TABLE columnar.chunk_group DROP CONSTRAINT chunk_group_storage_id_stripe_num_fkey;
11 | $;
12 | ELSE
13 | EXECUTE $
14 | ALTER TABLE columnar.chunk DROP CONSTRAINT chunk_storage_id_fkey;
15 | ALTER TABLE columnar.chunk_group DROP CONSTRAINT chunk_group_storage_id_fkey;
16 | $;
17 | END IF;
18 | END$proc$;
19 |
20 | -- since we dropped pg11 support, we don't need to worry about missing
21 | -- columnar objects when upgrading postgres
22 | DROP FUNCTION columnar.columnar_ensure_objects_exist();
23 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--10.1-1--10.2-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.1-1--10.2-1.sql
2 |
3 | -- For a proper mapping between tid & (stripe, row_num), add a new column to
4 | -- columnar.stripe and define a BTREE index on this column.
5 | -- Also include storage_id column for per-relation scans.
6 | ALTER TABLE columnar.stripe ADD COLUMN first_row_number bigint;
7 | CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number);
8 |
9 | -- Populate first_row_number column of columnar.stripe table.
10 | --
11 | -- For simplicity, we calculate MAX(row_count) value across all the stripes
12 | -- of all the columanar tables and then use it to populate first_row_number
13 | -- column. This would introduce some gaps however we are okay with that since
14 | -- it's already the case with regular INSERT/COPY's.
15 | DO $
16 | DECLARE
17 | max_row_count bigint;
18 | -- this should be equal to columnar_storage.h/COLUMNAR_FIRST_ROW_NUMBER
19 | COLUMNAR_FIRST_ROW_NUMBER constant bigint := 1;
20 | BEGIN
21 | SELECT MAX(row_count) INTO max_row_count FROM columnar.stripe;
22 | UPDATE columnar.stripe SET first_row_number = COLUMNAR_FIRST_ROW_NUMBER +
23 | (stripe_num - 1) * max_row_count;
24 | END;
25 | $;
26 |
27 | #include "udfs/upgrade_columnar_storage/10.2-1.sql"
28 | #include "udfs/downgrade_columnar_storage/10.2-1.sql"
29 |
30 | -- upgrade storage for all columnar relations
31 | SELECT columnar.upgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a
32 | WHERE c.relam = a.oid AND amname = 'columnar';
33 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--10.2-1--10.2-2.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.2-1--10.2-2.sql
2 |
3 | -- revoke read access for columnar.chunk from unprivileged
4 | -- user as it contains chunk min/max values
5 | REVOKE SELECT ON columnar.chunk FROM PUBLIC;
6 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--10.2-2--10.2-3.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.2-2--10.2-3.sql
2 |
3 | -- Since stripe_first_row_number_idx is required to scan a columnar table, we
4 | -- need to make sure that it is created before doing anything with columnar
5 | -- tables during pg upgrades.
6 | --
7 | -- However, a plain btree index is not a dependency of a table, so pg_upgrade
8 | -- cannot guarantee that stripe_first_row_number_idx gets created when
9 | -- creating columnar.stripe, unless we make it a unique "constraint".
10 | --
11 | -- To do that, drop stripe_first_row_number_idx and create a unique
12 | -- constraint with the same name to keep the code change at minimum.
13 | DROP INDEX columnar.stripe_first_row_number_idx;
14 | ALTER TABLE columnar.stripe ADD CONSTRAINT stripe_first_row_number_idx
15 | UNIQUE (storage_id, first_row_number);
16 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--10.2-3--10.2-4.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.2-3--10.2-4.sql
2 |
3 | #include "udfs/columnar_ensure_am_depends_catalog/10.2-4.sql"
4 |
5 | SELECT columnar.columnar_ensure_am_depends_catalog();
6 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--10.2-4--11.1-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.2.4--11.1-1.sql
2 |
3 | -- empty placeholder to allow standalone installation
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-1--11.1-2.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-1--11.2-1.sql
2 |
3 | #include "udfs/alter_table_set_access_method/11.1-2.sql"
4 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-10--11.1-11.sql:
--------------------------------------------------------------------------------
1 | #include "udfs/alter_table_set_access_method/11.1-11.sql"
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-11--11.1-12.sql:
--------------------------------------------------------------------------------
1 | #include "udfs/alter_table_set_access_method/11.1-12.sql"
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-2--11.1-3.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-1--11.2-1.sql
2 |
3 | #include "udfs/alter_columnar_table_set/11.1-3.sql"
4 | #include "udfs/alter_columnar_table_reset/11.1-3.sql"
5 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-4--11.1-5.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-4--11.1-5.sql
2 |
3 | #include "udfs/alter_table_set_access_method/11.1-5.sql"
4 |
5 | SET search_path TO columnar;
6 |
7 | CREATE SEQUENCE row_mask_seq START WITH 1 INCREMENT BY 1;
8 |
9 | CREATE TABLE row_mask (
10 | id BIGINT NOT NULL,
11 | storage_id BIGINT NOT NULL,
12 | stripe_id BIGINT NOT NULL,
13 | chunk_id INT NOT NULL,
14 | start_row_number BIGINT NOT NULL,
15 | end_row_number BIGINT NOT NULL,
16 | deleted_rows INT NOT NULL,
17 | mask BYTEA,
18 | PRIMARY KEY (id, storage_id, start_row_number, end_row_number)
19 | ) WITH (user_catalog_table = true);
20 |
21 | ALTER TABLE columnar.chunk_group ADD COLUMN deleted_rows BIGINT NOT NULL DEFAULT 0;
22 |
23 | ALTER TABLE columnar.row_mask ADD CONSTRAINT row_mask_stripe_unique
24 | UNIQUE (storage_id, stripe_id, start_row_number);
25 |
26 | ALTER TABLE columnar.row_mask ADD CONSTRAINT row_mask_chunk_unique
27 | UNIQUE (storage_id, stripe_id, chunk_id, start_row_number);
28 |
29 | REVOKE SELECT ON columnar.row_mask FROM PUBLIC;
30 |
31 | COMMENT ON TABLE row_mask IS 'Columnar chunk mask metadata';
32 |
33 | #include "udfs/create_table_row_mask/11.1-5.sql"
34 |
35 | -- find all columnar tables and create empty row mask for them
36 |
37 | SELECT columnar.create_table_row_mask(oid) FROM pg_class
38 | WHERE
39 | pg_class.relam = (SELECT oid FROM pg_am WHERE amname = 'columnar')
40 | AND
41 | pg_class.oid IN
42 | (
43 | SELECT (quote_ident(pg_tables.schemaname) || '.' ||
44 | quote_ident(pg_tables.tablename))::regclass::oid
45 | FROM pg_tables
46 | WHERE pg_tables.schemaname <> 'information_schema' AND
47 | pg_tables.schemaname <> 'pg_catalog' AND
48 | pg_tables.schemaname <> 'columnar'
49 | );
50 |
51 |
52 | RESET search_path;
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-5--11.1-6.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-5--11.1-6.sql
2 |
3 | #include "udfs/vacuum/11.1-6.sql"
4 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-6--11.1-7.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-6--11.1-7.sql
2 |
3 | #include "udfs/vacuum/11.1-7.sql"
4 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-7--11.1-8.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-6--11.1-7.sql
2 |
3 | #include "udfs/alter_table_set_access_method/11.1-8.sql"
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--11.1-9--11.1-10.sql:
--------------------------------------------------------------------------------
1 | DROP FUNCTION columnar.vacuum(REGCLASS, INT);
2 | DROP FUNCTION columnar.stats(
3 | IN regclass,
4 | OUT stripeId bigint,
5 | OUT fileOffset bigint,
6 | OUT rowCount integer,
7 | OUT deletedRows integer,
8 | OUT chunkCount integer,
9 | OUT dataLength integer
10 | );
11 | DROP FUNCTION columnar.vacuum_full(NAME, REAL, INT);
12 |
13 | #include "udfs/vacuum/11.1-10.sql"
14 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/columnar--9.5-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--9.5-1.sql
2 |
3 | -- empty placeholder to allow standalone installation
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--10.0-1--9.5-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.0-1--9.5-1.sql
2 |
3 | SET search_path TO columnar;
4 |
5 | DO $proc$
6 | BEGIN
7 |
8 | IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
9 | EXECUTE $
10 | DROP FUNCTION pg_catalog.alter_columnar_table_reset(
11 | table_name regclass,
12 | chunk_group_row_limit bool,
13 | stripe_row_limit bool,
14 | compression bool,
15 | compression_level bool);
16 |
17 | DROP FUNCTION pg_catalog.alter_columnar_table_set(
18 | table_name regclass,
19 | chunk_group_row_limit int,
20 | stripe_row_limit int,
21 | compression name,
22 | compression_level int);
23 |
24 | DROP ACCESS METHOD columnar;
25 |
26 | DROP FUNCTION columnar_handler(internal);
27 |
28 | $;
29 | END IF;
30 | END$proc$;
31 |
32 | DROP TABLE chunk;
33 | DROP TABLE chunk_group;
34 | DROP TABLE stripe;
35 | DROP TABLE options;
36 | DROP SEQUENCE storageid_seq;
37 |
38 | DROP FUNCTION columnar.columnar_ensure_objects_exist();
39 |
40 | RESET search_path;
41 | DROP SCHEMA columnar;
42 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--10.0-2--10.0-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.0-2--10.0-1.sql
2 |
3 | -- revoke read access for columnar metadata tables from unprivileged user
4 | REVOKE USAGE ON SCHEMA columnar FROM PUBLIC;
5 | REVOKE SELECT ON ALL tables IN SCHEMA columnar FROM PUBLIC;
6 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--10.1-1--10.0-3.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.1-1--10.0-3.sql
2 |
3 | -- define foreign keys between columnar metadata tables
4 | ALTER TABLE columnar.chunk
5 | ADD FOREIGN KEY (storage_id, stripe_num, chunk_group_num)
6 | REFERENCES columnar.chunk_group(storage_id, stripe_num, chunk_group_num) ON DELETE CASCADE;
7 |
8 | ALTER TABLE columnar.chunk_group
9 | ADD FOREIGN KEY (storage_id, stripe_num)
10 | REFERENCES columnar.stripe(storage_id, stripe_num) ON DELETE CASCADE;
11 |
12 | -- define columnar_ensure_objects_exist again
13 | #include "../udfs/columnar_ensure_objects_exist/10.0-1.sql"
14 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--10.2-1--10.1-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.2-1--10.1-1.sql
2 |
3 | -- downgrade storage for all columnar relations
4 | SELECT columnar.downgrade_columnar_storage(c.oid) FROM pg_class c, pg_am a
5 | WHERE c.relam = a.oid AND amname = 'columnar';
6 |
7 | DROP FUNCTION columnar.upgrade_columnar_storage(regclass);
8 | DROP FUNCTION columnar.downgrade_columnar_storage(regclass);
9 |
10 | -- drop "first_row_number" column and the index defined on it
11 | DROP INDEX columnar.stripe_first_row_number_idx;
12 | ALTER TABLE columnar.stripe DROP COLUMN first_row_number;
13 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--10.2-2--10.2-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.2-2--10.2-1.sql
2 |
3 | -- grant read access for columnar.chunk to unprivileged user
4 | GRANT SELECT ON columnar.chunk TO PUBLIC;
5 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--10.2-3--10.2-2.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.2-3--10.2-2.sql
2 |
3 | ALTER TABLE columnar.stripe DROP CONSTRAINT stripe_first_row_number_idx;
4 | CREATE INDEX stripe_first_row_number_idx ON columnar.stripe USING BTREE(storage_id, first_row_number);
5 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--10.2-4--10.2-3.sql:
--------------------------------------------------------------------------------
1 | -- columnar--10.2-4--10.2-3.sql
2 |
3 | DROP FUNCTION columnar.columnar_ensure_am_depends_catalog();
4 |
5 | -- Note that we intentionally do not delete pg_depend records that we inserted
6 | -- via columnar--10.2-3--10.2-4.sql (by using columnar_ensure_am_depends_catalog).
7 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--11.1-1--10.2-4.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-1--10.2-4.sql
2 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--11.1-2--11.1-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-2--11.1-1.sql
2 |
3 | DROP FUNCTION columnar.alter_table_set_access_method();
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--11.1-3--11.1-2.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-3--11.1-2.sql
2 |
3 | #include "udfs/alter_columnar_table_set/11.1-2.sql"
4 | #include "udfs/alter_columnar_table_reset/11.1-2.sql"
5 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/downgrades/columnar--11.1-5--11.1-4.sql:
--------------------------------------------------------------------------------
1 | -- columnar--11.1-4--11.1-4.sql
2 |
3 | DROP FUNCTION columnar.create_table_row_mask();
4 |
5 | DROP TABLE columnar.row_mask;
6 |
7 | DROP SEQUENCE columnar.row_mask_seq;
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/alter_columnar_table_reset/10.0-1.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
2 | table_name regclass,
3 | chunk_group_row_limit bool DEFAULT false,
4 | stripe_row_limit bool DEFAULT false,
5 | compression bool DEFAULT false,
6 | compression_level bool DEFAULT false)
7 | RETURNS void
8 | LANGUAGE C
9 | AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
10 |
11 | COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
12 | table_name regclass,
13 | chunk_group_row_limit bool,
14 | stripe_row_limit bool,
15 | compression bool,
16 | compression_level bool)
17 | IS 'reset on or more options on a columnar table to the system defaults';
18 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/alter_columnar_table_reset/11.1-2.sql:
--------------------------------------------------------------------------------
1 | DROP FUNCTION IF EXISTS columnar.alter_columnar_table_reset;
2 |
3 | CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_reset(
4 | table_name regclass,
5 | chunk_group_row_limit bool DEFAULT false,
6 | stripe_row_limit bool DEFAULT false,
7 | compression bool DEFAULT false,
8 | compression_level bool DEFAULT false)
9 | RETURNS void
10 | LANGUAGE C
11 | AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
12 |
13 | COMMENT ON FUNCTION pg_catalog.alter_columnar_table_reset(
14 | table_name regclass,
15 | chunk_group_row_limit bool,
16 | stripe_row_limit bool,
17 | compression bool,
18 | compression_level bool)
19 | IS 'reset on or more options on a columnar table to the system defaults';
20 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/alter_columnar_table_reset/11.1-3.sql:
--------------------------------------------------------------------------------
1 | DROP FUNCTION IF EXISTS pg_catalog.alter_columnar_table_reset;
2 |
3 | CREATE OR REPLACE FUNCTION columnar.alter_columnar_table_reset(
4 | table_name regclass,
5 | chunk_group_row_limit bool DEFAULT false,
6 | stripe_row_limit bool DEFAULT false,
7 | compression bool DEFAULT false,
8 | compression_level bool DEFAULT false)
9 | RETURNS void
10 | LANGUAGE C
11 | AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
12 |
13 | COMMENT ON FUNCTION columnar.alter_columnar_table_reset(
14 | table_name regclass,
15 | chunk_group_row_limit bool,
16 | stripe_row_limit bool,
17 | compression bool,
18 | compression_level bool)
19 | IS 'reset on or more options on a columnar table to the system defaults';
20 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/alter_columnar_table_reset/latest.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.alter_columnar_table_reset(
2 | table_name regclass,
3 | chunk_group_row_limit bool DEFAULT false,
4 | stripe_row_limit bool DEFAULT false,
5 | compression bool DEFAULT false,
6 | compression_level bool DEFAULT false)
7 | RETURNS void
8 | LANGUAGE C
9 | AS 'MODULE_PATHNAME', 'alter_columnar_table_reset';
10 |
11 | COMMENT ON FUNCTION columnar.alter_columnar_table_reset(
12 | table_name regclass,
13 | chunk_group_row_limit bool,
14 | stripe_row_limit bool,
15 | compression bool,
16 | compression_level bool)
17 | IS 'reset on or more options on a columnar table to the system defaults';
18 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/alter_columnar_table_set/10.0-1.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
2 | table_name regclass,
3 | chunk_group_row_limit int DEFAULT NULL,
4 | stripe_row_limit int DEFAULT NULL,
5 | compression name DEFAULT null,
6 | compression_level int DEFAULT NULL)
7 | RETURNS void
8 | LANGUAGE C
9 | AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
10 |
11 | COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
12 | table_name regclass,
13 | chunk_group_row_limit int,
14 | stripe_row_limit int,
15 | compression name,
16 | compression_level int)
17 | IS 'set one or more options on a columnar table, when set to NULL no change is made';
18 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/alter_columnar_table_set/11.1-2.sql:
--------------------------------------------------------------------------------
1 | DROP FUNCTION IF EXISTS columnar.alter_columnar_table_set;
2 |
3 | CREATE OR REPLACE FUNCTION pg_catalog.alter_columnar_table_set(
4 | table_name regclass,
5 | chunk_group_row_limit int DEFAULT NULL,
6 | stripe_row_limit int DEFAULT NULL,
7 | compression name DEFAULT null,
8 | compression_level int DEFAULT NULL)
9 | RETURNS void
10 | LANGUAGE C
11 | AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
12 |
13 | COMMENT ON FUNCTION pg_catalog.alter_columnar_table_set(
14 | table_name regclass,
15 | chunk_group_row_limit int,
16 | stripe_row_limit int,
17 | compression name,
18 | compression_level int)
19 | IS 'set one or more options on a columnar table, when set to NULL no change is made';
20 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/alter_columnar_table_set/11.1-3.sql:
--------------------------------------------------------------------------------
1 | DROP FUNCTION IF EXISTS pg_catalog.alter_columnar_table_set;
2 |
3 | CREATE OR REPLACE FUNCTION columnar.alter_columnar_table_set(
4 | table_name regclass,
5 | chunk_group_row_limit int DEFAULT NULL,
6 | stripe_row_limit int DEFAULT NULL,
7 | compression name DEFAULT null,
8 | compression_level int DEFAULT NULL)
9 | RETURNS void
10 | LANGUAGE C
11 | AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
12 |
13 | COMMENT ON FUNCTION columnar.alter_columnar_table_set(
14 | table_name regclass,
15 | chunk_group_row_limit int,
16 | stripe_row_limit int,
17 | compression name,
18 | compression_level int)
19 | IS 'set one or more options on a columnar table, when set to NULL no change is made';
20 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/alter_columnar_table_set/latest.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.alter_columnar_table_set(
2 | table_name regclass,
3 | chunk_group_row_limit int DEFAULT NULL,
4 | stripe_row_limit int DEFAULT NULL,
5 | compression name DEFAULT null,
6 | compression_level int DEFAULT NULL)
7 | RETURNS void
8 | LANGUAGE C
9 | AS 'MODULE_PATHNAME', 'alter_columnar_table_set';
10 |
11 | COMMENT ON FUNCTION columnar.alter_columnar_table_set(
12 | table_name regclass,
13 | chunk_group_row_limit int,
14 | stripe_row_limit int,
15 | compression name,
16 | compression_level int)
17 | IS 'set one or more options on a columnar table, when set to NULL no change is made';
18 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/10.2-4.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.columnar_ensure_am_depends_catalog()
2 | RETURNS void
3 | LANGUAGE plpgsql
4 | SET search_path = pg_catalog
5 | AS $func$
6 | BEGIN
7 | INSERT INTO pg_depend
8 | SELECT -- Define a dependency edge from "columnar table access method" ..
9 | 'pg_am'::regclass::oid as classid,
10 | (select oid from pg_am where amname = 'columnar') as objid,
11 | 0 as objsubid,
12 | -- ... to each object that is registered to pg_class and that lives
13 | -- in "columnar" schema. That contains catalog tables, indexes
14 | -- created on them and the sequences created in "columnar" schema.
15 | --
16 | -- Given the possibility of user might have created their own objects
17 | -- in columnar schema, we explicitly specify list of objects that we
18 | -- are interested in.
19 | 'pg_class'::regclass::oid as refclassid,
20 | columnar_schema_members.relname::regclass::oid as refobjid,
21 | 0 as refobjsubid,
22 | 'n' as deptype
23 | FROM (VALUES ('columnar.chunk'),
24 | ('columnar.chunk_group'),
25 | ('columnar.chunk_group_pkey'),
26 | ('columnar.chunk_pkey'),
27 | ('columnar.options'),
28 | ('columnar.options_pkey'),
29 | ('columnar.storageid_seq'),
30 | ('columnar.stripe'),
31 | ('columnar.stripe_first_row_number_idx'),
32 | ('columnar.stripe_pkey')
33 | ) columnar_schema_members(relname)
34 | -- Avoid inserting duplicate entries into pg_depend.
35 | EXCEPT TABLE pg_depend;
36 | END;
37 | $func$;
38 | COMMENT ON FUNCTION columnar.columnar_ensure_am_depends_catalog()
39 | IS 'internal function responsible for creating dependencies from columnar '
40 | 'table access method to the rel objects in columnar schema';
41 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/columnar_ensure_am_depends_catalog/latest.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.columnar_ensure_am_depends_catalog()
2 | RETURNS void
3 | LANGUAGE plpgsql
4 | SET search_path = pg_catalog
5 | AS $func$
6 | BEGIN
7 | INSERT INTO pg_depend
8 | SELECT -- Define a dependency edge from "columnar table access method" ..
9 | 'pg_am'::regclass::oid as classid,
10 | (select oid from pg_am where amname = 'columnar') as objid,
11 | 0 as objsubid,
12 | -- ... to each object that is registered to pg_class and that lives
13 | -- in "columnar" schema. That contains catalog tables, indexes
14 | -- created on them and the sequences created in "columnar" schema.
15 | --
16 | -- Given the possibility of user might have created their own objects
17 | -- in columnar schema, we explicitly specify list of objects that we
18 | -- are interested in.
19 | 'pg_class'::regclass::oid as refclassid,
20 | columnar_schema_members.relname::regclass::oid as refobjid,
21 | 0 as refobjsubid,
22 | 'n' as deptype
23 | FROM (VALUES ('columnar.chunk'),
24 | ('columnar.chunk_group'),
25 | ('columnar.chunk_group_pkey'),
26 | ('columnar.chunk_pkey'),
27 | ('columnar.options'),
28 | ('columnar.options_pkey'),
29 | ('columnar.storageid_seq'),
30 | ('columnar.stripe'),
31 | ('columnar.stripe_first_row_number_idx'),
32 | ('columnar.stripe_pkey')
33 | ) columnar_schema_members(relname)
34 | -- Avoid inserting duplicate entries into pg_depend.
35 | EXCEPT TABLE pg_depend;
36 | END;
37 | $func$;
38 | COMMENT ON FUNCTION columnar.columnar_ensure_am_depends_catalog()
39 | IS 'internal function responsible for creating dependencies from columnar '
40 | 'table access method to the rel objects in columnar schema';
41 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/10.0-1.sql:
--------------------------------------------------------------------------------
1 | -- columnar.columnar_ensure_objects_exist is an internal helper function to create
2 | -- missing objects, anything related to the table access methods.
3 | -- Since the API for table access methods is only available in PG12 we can't create these
4 | -- objects when Citus is installed in PG11. Once citus is installed on PG11 the user can
5 | -- upgrade their database to PG12. Now they require the table access method objects that
6 | -- we couldn't create before.
7 | -- This internal function is called from `citus_finish_pg_upgrade` which the user is
8 | -- required to call after a PG major upgrade.
9 | CREATE OR REPLACE FUNCTION columnar.columnar_ensure_objects_exist()
10 | RETURNS void
11 | LANGUAGE plpgsql
12 | SET search_path = pg_catalog
13 | AS $ceoe$
14 | BEGIN
15 |
16 | -- when postgres is version 12 or above we need to create the tableam. If the tableam
17 | -- exist we assume all objects have been created.
18 | IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
19 | IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN
20 |
21 | #include "../columnar_handler/10.0-1.sql"
22 |
23 | #include "../alter_columnar_table_set/10.0-1.sql"
24 |
25 | #include "../alter_columnar_table_reset/10.0-1.sql"
26 |
27 | -- add the missing objects to the extension
28 | ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler(internal);
29 | ALTER EXTENSION citus ADD ACCESS METHOD columnar;
30 | ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set(
31 | table_name regclass,
32 | chunk_group_row_limit int,
33 | stripe_row_limit int,
34 | compression name,
35 | compression_level int);
36 | ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset(
37 | table_name regclass,
38 | chunk_group_row_limit bool,
39 | stripe_row_limit bool,
40 | compression bool,
41 | compression_level bool);
42 |
43 | END IF;
44 | END IF;
45 | END;
46 | $ceoe$;
47 |
48 | COMMENT ON FUNCTION columnar.columnar_ensure_objects_exist()
49 | IS 'internal function to be called by pg_catalog.citus_finish_pg_upgrade responsible for creating the columnar objects';
50 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/columnar_ensure_objects_exist/latest.sql:
--------------------------------------------------------------------------------
1 | -- columnar.columnar_ensure_objects_exist is an internal helper function to create
2 | -- missing objects, anything related to the table access methods.
3 | -- Since the API for table access methods is only available in PG12 we can't create these
4 | -- objects when Citus is installed in PG11. Once citus is installed on PG11 the user can
5 | -- upgrade their database to PG12. Now they require the table access method objects that
6 | -- we couldn't create before.
7 | -- This internal function is called from `citus_finish_pg_upgrade` which the user is
8 | -- required to call after a PG major upgrade.
9 | CREATE OR REPLACE FUNCTION columnar.columnar_ensure_objects_exist()
10 | RETURNS void
11 | LANGUAGE plpgsql
12 | SET search_path = pg_catalog
13 | AS $ceoe$
14 | BEGIN
15 |
16 | -- when postgres is version 12 or above we need to create the tableam. If the tableam
17 | -- exist we assume all objects have been created.
18 | IF substring(current_Setting('server_version'), '\d+')::int >= 12 THEN
19 | IF NOT EXISTS (SELECT 1 FROM pg_am WHERE amname = 'columnar') THEN
20 |
21 | #include "../columnar_handler/10.0-1.sql"
22 |
23 | #include "../alter_columnar_table_set/10.0-1.sql"
24 |
25 | #include "../alter_columnar_table_reset/10.0-1.sql"
26 |
27 | -- add the missing objects to the extension
28 | ALTER EXTENSION citus ADD FUNCTION columnar.columnar_handler(internal);
29 | ALTER EXTENSION citus ADD ACCESS METHOD columnar;
30 | ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_set(
31 | table_name regclass,
32 | chunk_group_row_limit int,
33 | stripe_row_limit int,
34 | compression name,
35 | compression_level int);
36 | ALTER EXTENSION citus ADD FUNCTION pg_catalog.alter_columnar_table_reset(
37 | table_name regclass,
38 | chunk_group_row_limit bool,
39 | stripe_row_limit bool,
40 | compression bool,
41 | compression_level bool);
42 |
43 | END IF;
44 | END IF;
45 | END;
46 | $ceoe$;
47 |
48 | COMMENT ON FUNCTION columnar.columnar_ensure_objects_exist()
49 | IS 'internal function to be called by pg_catalog.citus_finish_pg_upgrade responsible for creating the columnar objects';
50 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/columnar_handler/10.0-1.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal)
2 | RETURNS table_am_handler
3 | LANGUAGE C
4 | AS 'MODULE_PATHNAME', 'columnar_handler';
5 |
6 | COMMENT ON FUNCTION columnar.columnar_handler(internal)
7 | IS 'internal function returning the handler for columnar tables';
8 |
9 | -- postgres 11.8 does not support the syntax for table am, also it is seemingly trying
10 | -- to parse the upgrade file and erroring on unknown syntax.
11 | -- normally this section would not execute on postgres 11 anyway. To trick it to pass on
12 | -- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid
13 | -- syntax on 11.8 and will execute correctly in 12
14 | DO $create_table_am$
15 | BEGIN
16 | EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler';
17 | END $create_table_am$;
18 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/columnar_handler/latest.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.columnar_handler(internal)
2 | RETURNS table_am_handler
3 | LANGUAGE C
4 | AS 'MODULE_PATHNAME', 'columnar_handler';
5 |
6 | COMMENT ON FUNCTION columnar.columnar_handler(internal)
7 | IS 'internal function returning the handler for columnar tables';
8 |
9 | -- postgres 11.8 does not support the syntax for table am, also it is seemingly trying
10 | -- to parse the upgrade file and erroring on unknown syntax.
11 | -- normally this section would not execute on postgres 11 anyway. To trick it to pass on
12 | -- 11.8 we wrap the statement in a plpgsql block together with an EXECUTE. This is valid
13 | -- syntax on 11.8 and will execute correctly in 12
14 | DO $create_table_am$
15 | BEGIN
16 | EXECUTE 'CREATE ACCESS METHOD columnar TYPE TABLE HANDLER columnar.columnar_handler';
17 | END $create_table_am$;
18 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/create_table_row_mask/11.1-5.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.create_table_row_mask(
2 | table_name regclass)
3 | RETURNS bool
4 | LANGUAGE C
5 | AS 'MODULE_PATHNAME', 'create_table_row_mask';
6 |
7 | COMMENT ON FUNCTION columnar.create_table_row_mask(
8 | table_name regclass)
9 | IS 'Create empty row mask for table';
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/create_table_row_mask/latest.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.create_table_row_mask(
2 | table_name regclass)
3 | RETURNS bool
4 | LANGUAGE C
5 | AS 'MODULE_PATHNAME', 'create_table_row_mask';
6 |
7 | COMMENT ON FUNCTION columnar.create_table_row_mask(
8 | table_name regclass)
9 | IS 'Create empty row mask for table';
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/downgrade_columnar_storage/10.2-1.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.downgrade_columnar_storage(rel regclass)
2 | RETURNS VOID
3 | STRICT
4 | LANGUAGE c AS 'MODULE_PATHNAME', $downgrade_columnar_storage$;
5 |
6 | COMMENT ON FUNCTION columnar.downgrade_columnar_storage(regclass)
7 | IS 'function to downgrade the columnar storage, if necessary';
8 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/downgrade_columnar_storage/latest.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.downgrade_columnar_storage(rel regclass)
2 | RETURNS VOID
3 | STRICT
4 | LANGUAGE c AS 'MODULE_PATHNAME', $downgrade_columnar_storage$;
5 |
6 | COMMENT ON FUNCTION columnar.downgrade_columnar_storage(regclass)
7 | IS 'function to downgrade the columnar storage, if necessary';
8 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/upgrade_columnar_storage/10.2-1.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.upgrade_columnar_storage(rel regclass)
2 | RETURNS VOID
3 | STRICT
4 | LANGUAGE c AS 'MODULE_PATHNAME', $upgrade_columnar_storage$;
5 |
6 | COMMENT ON FUNCTION columnar.upgrade_columnar_storage(regclass)
7 | IS 'function to upgrade the columnar storage, if necessary';
8 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/upgrade_columnar_storage/latest.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.upgrade_columnar_storage(rel regclass)
2 | RETURNS VOID
3 | STRICT
4 | LANGUAGE c AS 'MODULE_PATHNAME', $upgrade_columnar_storage$;
5 |
6 | COMMENT ON FUNCTION columnar.upgrade_columnar_storage(regclass)
7 | IS 'function to upgrade the columnar storage, if necessary';
8 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/vacuum/11.1-10.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar._vacuum_internal(rel REGCLASS, stripe_count INT)
2 | RETURNS INT
3 | LANGUAGE c AS 'MODULE_PATHNAME', $vacuum_columnar_table$;
4 |
5 | COMMENT ON FUNCTION columnar._vacuum_internal(REGCLASS, INT)
6 | IS 'vacuum columnar table internal function';
7 |
8 | CREATE OR REPLACE FUNCTION columnar.vacuum(tablename REGCLASS, stripe_count INT DEFAULT 0)
9 | RETURNS INT AS $
10 | DECLARE
11 | count INT;
12 | stripes INT;
13 | BEGIN
14 | count := 1;
15 | stripes := 0;
16 |
17 | WHILE count > 0 AND (stripe_count = 0 OR stripes < stripe_count) LOOP
18 | SELECT columnar._vacuum_internal(tablename, stripe_count) INTO count;
19 | stripes := stripes + count;
20 | END LOOP;
21 |
22 | return stripes;
23 | END;
24 | $ LANGUAGE plpgsql;
25 |
26 | COMMENT ON FUNCTION columnar.vacuum(REGCLASS, INT)
27 | IS 'vacuum columnar table function';
28 |
29 | CREATE OR REPLACE FUNCTION columnar.stats(
30 | IN regclass,
31 | OUT stripeId bigint,
32 | OUT fileOffset bigint,
33 | OUT rowCount integer,
34 | OUT deletedRows integer,
35 | OUT chunkCount integer,
36 | OUT dataLength integer
37 | ) RETURNS SETOF record
38 | LANGUAGE c
39 | AS 'MODULE_PATHNAME', $columnar_stats$;
40 |
41 | COMMENT ON FUNCTION columnar.stats(regclass)
42 | IS 'columnar stripe statistics';
43 |
44 | CREATE OR REPLACE FUNCTION columnar.vacuum_full(schema NAME DEFAULT 'public', sleep_time REAL DEFAULT .1, stripe_count INT DEFAULT 25)
45 | RETURNS VOID AS $
46 | DECLARE
47 | tables REGCLASS[];
48 | tablename REGCLASS;
49 | finished BOOL;
50 | count INT;
51 | BEGIN
52 | SELECT ARRAY_AGG(c.relname) INTO tables
53 | FROM pg_catalog.pg_class c
54 | LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
55 | LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam
56 | WHERE c.relkind = 'r'
57 | AND n.nspname <> 'pg_catalog'
58 | AND n.nspname !~ '^pg_toast'
59 | AND n.nspname <> 'information_schema'
60 | AND pg_catalog.pg_table_is_visible(c.oid)
61 | AND am.amname = 'columnar'
62 | AND n.nspname = schema
63 | ORDER BY 1;
64 |
65 | FOREACH tablename IN ARRAY tables
66 | LOOP
67 | finished := 'f';
68 | count := 1;
69 | WHILE count > 0 LOOP
70 | SELECT columnar.vacuum(tablename, stripe_count) INTO count;
71 | PERFORM pg_sleep(sleep_time);
72 | END LOOP;
73 | END LOOP;
74 | END;
75 | $ LANGUAGE plpgsql;
76 |
77 | COMMENT ON FUNCTION columnar.vacuum_full(name, real, int)
78 | IS 'vacuum columnar schema in full incrementally';
79 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/vacuum/11.1-6.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar.vacuum(rel regclass, optional_count int DEFAULT 0)
2 | RETURNS INT
3 | LANGUAGE c AS 'MODULE_PATHNAME', $vacuum_columnar_table$;
4 |
5 | COMMENT ON FUNCTION columnar.vacuum(regclass, int)
6 | IS 'vacuum columnar table';
7 |
8 | CREATE OR REPLACE FUNCTION columnar.stats(
9 | IN regclass,
10 | OUT stripeId bigint,
11 | OUT fileOffset bigint,
12 | OUT rowCount integer,
13 | OUT deletedRows integer,
14 | OUT chunkCount integer,
15 | OUT dataLength integer
16 | ) RETURNS SETOF record
17 | LANGUAGE c
18 | AS 'MODULE_PATHNAME', $columnar_stats$;
19 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/vacuum/11.1-7.sql:
--------------------------------------------------------------------------------
1 | COMMENT ON FUNCTION columnar.stats(regclass)
2 | IS 'columnar stripe statistics';
3 |
4 | CREATE OR REPLACE FUNCTION columnar.vacuum_full(schema NAME DEFAULT 'public', sleep_time REAL DEFAULT .1, stripe_count INT DEFAULT 25)
5 | RETURNS VOID AS $
6 | DECLARE
7 | tables REGCLASS[];
8 | tablename REGCLASS;
9 | finished BOOL;
10 | count INT;
11 | BEGIN
12 | SELECT ARRAY_AGG(c.relname) INTO tables
13 | FROM pg_catalog.pg_class c
14 | LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
15 | LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam
16 | WHERE c.relkind = 'r'
17 | AND n.nspname <> 'pg_catalog'
18 | AND n.nspname !~ '^pg_toast'
19 | AND n.nspname <> 'information_schema'
20 | AND pg_catalog.pg_table_is_visible(c.oid)
21 | AND am.amname = 'columnar'
22 | AND n.nspname = schema
23 | ORDER BY 1;
24 |
25 | FOREACH tablename IN ARRAY tables
26 | LOOP
27 | finished := 'f';
28 | count := 1;
29 | WHILE count > 0 LOOP
30 | SELECT columnar.vacuum(tablename, stripe_count) INTO count;
31 | PERFORM pg_sleep(sleep_time);
32 | END LOOP;
33 | END LOOP;
34 | END;
35 | $ LANGUAGE plpgsql;
36 |
37 | COMMENT ON FUNCTION columnar.vacuum_full(name, real, int)
38 | IS 'vacuum columnar schema in full incrementally';
39 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/sql/udfs/vacuum/latest.sql:
--------------------------------------------------------------------------------
1 | CREATE OR REPLACE FUNCTION columnar._vacuum_internal(rel REGCLASS, stripe_count INT)
2 | RETURNS INT
3 | LANGUAGE c AS 'MODULE_PATHNAME', $vacuum_columnar_table$;
4 |
5 | COMMENT ON FUNCTION columnar._vacuum_internal(REGCLASS, INT)
6 | IS 'vacuum columnar table internal function';
7 |
8 | CREATE OR REPLACE FUNCTION columnar.vacuum(tablename REGCLASS, stripe_count INT DEFAULT 0)
9 | RETURNS INT AS $
10 | DECLARE
11 | count INT;
12 | stripes INT;
13 | BEGIN
14 | count := 1;
15 | stripes := 0;
16 |
17 | WHILE count > 0 AND (stripe_count = 0 OR stripes < stripe_count) LOOP
18 | SELECT columnar._vacuum_internal(tablename, stripe_count) INTO count;
19 | stripes := stripes + count;
20 | END LOOP;
21 |
22 | return stripes;
23 | END;
24 | $ LANGUAGE plpgsql;
25 |
26 | COMMENT ON FUNCTION columnar.vacuum(REGCLASS, INT)
27 | IS 'vacuum columnar table function';
28 |
29 | CREATE OR REPLACE FUNCTION columnar.stats(
30 | IN regclass,
31 | OUT stripeId bigint,
32 | OUT fileOffset bigint,
33 | OUT rowCount integer,
34 | OUT deletedRows integer,
35 | OUT chunkCount integer,
36 | OUT dataLength integer
37 | ) RETURNS SETOF record
38 | LANGUAGE c
39 | AS 'MODULE_PATHNAME', $columnar_stats$;
40 |
41 | COMMENT ON FUNCTION columnar.stats(regclass)
42 | IS 'columnar stripe statistics';
43 |
44 | CREATE OR REPLACE FUNCTION columnar.vacuum_full(schema NAME DEFAULT 'public', sleep_time REAL DEFAULT .1, stripe_count INT DEFAULT 25)
45 | RETURNS VOID AS $
46 | DECLARE
47 | tables REGCLASS[];
48 | tablename REGCLASS;
49 | finished BOOL;
50 | count INT;
51 | BEGIN
52 | SELECT ARRAY_AGG(c.relname) INTO tables
53 | FROM pg_catalog.pg_class c
54 | LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
55 | LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam
56 | WHERE c.relkind = 'r'
57 | AND n.nspname <> 'pg_catalog'
58 | AND n.nspname !~ '^pg_toast'
59 | AND n.nspname <> 'information_schema'
60 | AND pg_catalog.pg_table_is_visible(c.oid)
61 | AND am.amname = 'columnar'
62 | AND n.nspname = schema
63 | ORDER BY 1;
64 |
65 | FOREACH tablename IN ARRAY tables
66 | LOOP
67 | finished := 'f';
68 | count := 1;
69 | WHILE count > 0 LOOP
70 | SELECT columnar.vacuum(tablename, stripe_count) INTO count;
71 | PERFORM pg_sleep(sleep_time);
72 | END LOOP;
73 | END LOOP;
74 | END;
75 | $ LANGUAGE plpgsql;
76 |
77 | COMMENT ON FUNCTION columnar.vacuum_full(name, real, int)
78 | IS 'vacuum columnar schema in full incrementally';
79 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/vectorization/types/date.c:
--------------------------------------------------------------------------------
1 |
2 | #include "postgres.h"
3 |
4 | #include "nodes/execnodes.h"
5 | #include "utils/date.h"
6 |
7 | #include "columnar/vectorization/types/types.h"
8 |
9 | // date (int32)
10 | BUILD_CMP_OPERATOR_INT(date_, DateADT, DateADT)
11 |
12 | // time (int64)
13 | BUILD_CMP_OPERATOR_INT(time_, TimeADT, TimeADT)
14 |
15 | // timestamp (int64)
16 | BUILD_CMP_OPERATOR_INT(timestamp_, Timestamp, Timestamp)
17 |
18 | // timestamptz (int64)
19 | BUILD_CMP_OPERATOR_INT(timestamptz_, TimestampTz, TimestampTz)
20 |
--------------------------------------------------------------------------------
/columnar/src/backend/columnar/vectorization/types/int.c:
--------------------------------------------------------------------------------
1 |
2 | #include "postgres.h"
3 | #include "common/int.h"
4 |
5 | #include "fmgr.h"
6 | #include "nodes/execnodes.h"
7 |
8 | #include "columnar/vectorization/types/types.h"
9 |
10 | // char
11 | BUILD_CMP_OPERATOR_INT( char, char, char)
12 |
13 | // int2
14 | BUILD_CMP_OPERATOR_INT( int2, int16, int16)
15 | BUILD_CMP_OPERATOR_INT(int24, int16, int32)
16 | BUILD_CMP_OPERATOR_INT(int28, int16, int64)
17 |
18 | // int4
19 | BUILD_CMP_OPERATOR_INT( int4, int32, int32)
20 | BUILD_CMP_OPERATOR_INT(int42, int32, int16)
21 | BUILD_CMP_OPERATOR_INT(int48, int32, int64)
22 |
23 | // int8
24 | BUILD_CMP_OPERATOR_INT( int8, int64, int64)
25 | BUILD_CMP_OPERATOR_INT(int82, int64, int16)
26 | BUILD_CMP_OPERATOR_INT(int84, int64, int32)
27 |
--------------------------------------------------------------------------------
/columnar/src/include/.gitignore:
--------------------------------------------------------------------------------
1 | /stamp-h
2 | /stamp-ext-h
3 | /citus_config.h
4 | /citus_config.h.in~
5 | /citus_version.h
6 | /citus_version.h.in~
7 |
--------------------------------------------------------------------------------
/columnar/src/include/citus_version.h.in:
--------------------------------------------------------------------------------
1 | /* This file is created manually */
2 |
3 | /* Citus full name as a string */
4 | #undef CITUS_NAME
5 |
6 | /* Citus edition as a string */
7 | #undef CITUS_EDITION
8 |
9 | /* Extension version expected by this Citus build */
10 | #undef CITUS_EXTENSIONVERSION
11 |
12 | /* Citus major version as a string */
13 | #undef CITUS_MAJORVERSION
14 |
15 | /* Citus version as a string */
16 | #undef CITUS_VERSION
17 |
18 | /* Citus version as a number */
19 | #undef CITUS_VERSION_NUM
20 |
21 | /* A string containing the version number, platform, and C compiler */
22 | #undef CITUS_VERSION_STR
23 |
24 | /* Define to 1 if you have the `curl' library (-lcurl). */
25 | #undef HAVE_LIBCURL
26 |
27 | /* Define to 1 if you have the `liblz4' library (-llz4). */
28 | #undef HAVE_CITUS_LIBLZ4
29 |
30 | /* Define to 1 if you have the `libzstd' library (-lzstd). */
31 | #undef HAVE_LIBZSTD
32 |
33 | /* Base URL for statistics collection and update checks */
34 | #undef REPORTS_BASE_URL
35 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/columnar_compression.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_compression.h
4 | *
5 | * Type and function declarations for compression methods.
6 | *
7 | * Copyright (c) Citus Data, Inc.
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 | #ifndef COLUMNAR_COMPRESSION_H
13 | #define COLUMNAR_COMPRESSION_H
14 |
15 | /* Enumaration for columnar table's compression method */
16 | typedef enum
17 | {
18 | COMPRESSION_TYPE_INVALID = -1,
19 | COMPRESSION_NONE = 0,
20 | COMPRESSION_PG_LZ = 1,
21 | COMPRESSION_LZ4 = 2,
22 | COMPRESSION_ZSTD = 3,
23 |
24 | COMPRESSION_COUNT
25 | } CompressionType;
26 |
27 | extern bool CompressBuffer(StringInfo inputBuffer,
28 | StringInfo outputBuffer,
29 | CompressionType compressionType,
30 | int compressionLevel);
31 | extern StringInfo DecompressBuffer(StringInfo buffer, CompressionType compressionType,
32 | uint64 decompressedSize);
33 |
34 | #endif /* COLUMNAR_COMPRESSION_H */
35 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/columnar_customscan.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_customscan.h
4 | *
5 | * Forward declarations of functions to hookup the custom scan feature of
6 | * columnar.
7 | *
8 | * $Id$
9 | *
10 | *-------------------------------------------------------------------------
11 | */
12 |
13 | #ifndef COLUMNAR_CUSTOMSCAN_H
14 | #define COLUMNAR_CUSTOMSCAN_H
15 |
16 | #include "nodes/extensible.h"
17 |
18 | /* Flag to indicate is vectorized aggregate used in execution */
19 | #define CUSTOM_SCAN_VECTORIZED_AGGREGATE 1
20 |
21 | extern void columnar_customscan_init(void);
22 | extern const CustomScanMethods * columnar_customscan_methods(void);
23 | extern Bitmapset * ColumnarAttrNeeded(ScanState *ss, List *customList);
24 |
25 | #endif /* COLUMNAR_CUSTOMSCAN_H */
26 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/columnar_indexscan.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_indexscan.h
4 | * Custom scan method for index
5 | *
6 | * IDENTIFICATION
7 | * src/backend/columnar/columnar_indexscan.c
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 |
13 | #ifndef COLUMNAR_INDEXSCAN_H
14 | #define COLUMNAR_INDEXSCAN_H
15 |
16 | #include "postgres.h"
17 |
18 | #include "nodes/execnodes.h"
19 |
20 | typedef struct ColumnarIndexScanState
21 | {
22 | CustomScanState css;
23 | IndexScanState *indexscan_state;
24 | } ColumnarIndexScanState;
25 |
26 | extern CustomScan * columnar_create_indexscan_node(void);
27 | extern void columnar_register_indexscan_node(void);
28 |
29 | #endif
--------------------------------------------------------------------------------
/columnar/src/include/columnar/columnar_metadata.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_metadata.h
4 | *
5 | * Type and function declarations for Columnar metadata.
6 | *
7 | * Copyright (c) Citus Data, Inc.
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 | #ifndef COLUMNAR_METADATA_H
13 | #define COLUMNAR_METADATA_H
14 |
15 | #include "access/sdir.h"
16 |
17 | /*
18 | * StripeMetadata represents information about a stripe. This information is
19 | * stored in the metadata table "columnar.stripe".
20 | */
21 | typedef struct StripeMetadata
22 | {
23 | uint64 fileOffset;
24 | uint64 dataLength;
25 | uint32 columnCount;
26 | uint32 chunkCount;
27 | uint32 chunkGroupRowCount;
28 | uint64 rowCount;
29 | uint64 id;
30 | uint64 firstRowNumber;
31 |
32 | /* see StripeWriteState */
33 | bool aborted;
34 |
35 | /*
36 | * If write operation is in-progress (i.e. StripeWriteState returned
37 | * STRIPE_WRITE_IN_PROGRESS), then insertedByCurrentXact is used to
38 | * distinguish whether it's being written by current transaction or
39 | * not.
40 | */
41 | bool insertedByCurrentXact;
42 | } StripeMetadata;
43 |
44 | /*
45 | * EmptyStripeReservation represents information for an empty stripe
46 | * reservation.
47 | */
48 | typedef struct EmptyStripeReservation
49 | {
50 | uint64 stripeId;
51 | uint64 stripeFirstRowNumber;
52 | } EmptyStripeReservation;
53 |
54 | extern List * StripesForRelfilenode(RelFileLocator relfilelocator, ScanDirection scanDirection);
55 | extern uint32 DeletedRowsForStripe(RelFileLocator relfilelocator,
56 | uint32 chunkCount,
57 | uint64 stripeId);
58 | extern Size DecompressedLengthForStripe(RelFileLocator relfilelocator, uint64 stripeId);
59 | extern void ColumnarStorageUpdateIfNeeded(Relation rel, bool isUpgrade);
60 | extern StripeMetadata * RewriteStripeMetadataRowWithNewValues(Relation rel, uint64 stripeId,
61 | uint64 sizeBytes, uint64 fileOffset, uint64 rowCount, uint64 chunkCount);
62 |
63 | #endif /* COLUMNAR_METADATA_H */
64 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/columnar_storage.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_storage.h
4 | *
5 | * Type and function declarations for storage of columnar data in blocks.
6 | *
7 | * Copyright (c) Citus Data, Inc.
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 | #ifndef COLUMNAR_STORAGE_H
13 | #define COLUMNAR_STORAGE_H
14 |
15 | #include "postgres.h"
16 |
17 | #include "storage/smgr.h"
18 | #include "utils/rel.h"
19 |
20 | #include "columnar/columnar_tableam.h"
21 |
22 |
23 | #define COLUMNAR_INVALID_ROW_NUMBER ((uint64) 0)
24 | #define COLUMNAR_FIRST_ROW_NUMBER ((uint64) 1)
25 | #define COLUMNAR_MAX_ROW_NUMBER ((uint64) \
26 | (COLUMNAR_FIRST_ROW_NUMBER + \
27 | VALID_ITEMPOINTER_OFFSETS * \
28 | VALID_BLOCKNUMBERS))
29 |
30 |
31 | /*
32 | * Logical offsets never fall on the first two physical pages. See
33 | * comments in columnar_storage.c.
34 | */
35 | #define ColumnarInvalidLogicalOffset 0
36 | #define ColumnarFirstLogicalOffset ((BLCKSZ - SizeOfPageHeaderData) * 2)
37 | #define ColumnarLogicalOffsetIsValid(X) ((X) >= ColumnarFirstLogicalOffset)
38 |
39 |
40 | extern void ColumnarStorageInit(SMgrRelation srel, uint64 storageId);
41 | extern bool ColumnarStorageIsCurrent(Relation rel);
42 | extern void ColumnarStorageUpdateCurrent(Relation rel, bool upgrade,
43 | uint64 reservedStripeId,
44 | uint64 reservedRowNumber,
45 | uint64 reservedOffset);
46 |
47 | extern uint64 ColumnarStorageGetVersionMajor(Relation rel, bool force);
48 | extern uint64 ColumnarStorageGetVersionMinor(Relation rel, bool force);
49 | extern uint64 ColumnarStorageGetStorageId(Relation rel, bool force);
50 | extern uint64 ColumnarStorageGetReservedStripeId(Relation rel, bool force);
51 | extern uint64 ColumnarStorageGetReservedRowNumber(Relation rel, bool force);
52 | extern uint64 ColumnarStorageGetReservedOffset(Relation rel, bool force);
53 |
54 | extern uint64 ColumnarStorageReserveData(Relation rel, uint64 amount);
55 | extern uint64 ColumnarStorageReserveRowNumber(Relation rel, uint64 nrows);
56 | extern uint64 ColumnarStorageReserveStripeId(Relation rel);
57 |
58 | extern void ColumnarStorageRead(Relation rel, uint64 logicalOffset,
59 | char *data, uint32 amount);
60 | extern void ColumnarStorageWrite(Relation rel, uint64 logicalOffset,
61 | char *data, uint32 amount);
62 | extern bool ColumnarStorageTruncate(Relation rel, uint64 newDataReservation);
63 |
64 | #endif /* COLUMNAR_STORAGE_H */
65 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/columnar_tableam.h:
--------------------------------------------------------------------------------
1 | #ifndef COLUMNAR_TABLEAM_H
2 | #define COLUMNAR_TABLEAM_H
3 | #include "citus_version.h"
4 |
5 | #include "postgres.h"
6 | #include "fmgr.h"
7 | #include "access/tableam.h"
8 | #include "access/skey.h"
9 | #include "nodes/bitmapset.h"
10 | #include "access/heapam.h"
11 | #include "catalog/indexing.h"
12 | #include "utils/acl.h"
13 |
14 | #include "columnar/columnar.h"
15 |
16 | /*
17 | * Number of valid ItemPointer Offset's for "row number" <> "ItemPointer"
18 | * mapping.
19 | *
20 | * Postgres has some asserts calling either ItemPointerIsValid or
21 | * OffsetNumberIsValid. That constraints itemPointer.offsetNumber
22 | * for columnar tables to the following interval:
23 | * [FirstOffsetNumber, MaxOffsetNumber].
24 | *
25 | * However, bitmap scan logic assumes that itemPointer.offsetNumber cannot
26 | * be larger than MaxHeapTuplesPerPage (see tbm_add_tuples).
27 | *
28 | * For this reason, we restrict itemPointer.offsetNumber
29 | * to the following interval: [FirstOffsetNumber, MaxHeapTuplesPerPage].
30 | */
31 | #define VALID_ITEMPOINTER_OFFSETS \
32 | ((uint64) (MaxHeapTuplesPerPage - FirstOffsetNumber + 1))
33 |
34 | /*
35 | * Number of valid ItemPointer BlockNumber's for "row number" <> "ItemPointer"
36 | * mapping.
37 | *
38 | * Similar to VALID_ITEMPOINTER_OFFSETS, due to asserts around
39 | * itemPointer.blockNumber, we can only use values upto and including
40 | * MaxBlockNumber.
41 | * Note that postgres doesn't restrict blockNumber to a lower boundary.
42 | *
43 | * For this reason, we restrict itemPointer.blockNumber
44 | * to the following interval: [0, MaxBlockNumber].
45 | */
46 | #define VALID_BLOCKNUMBERS ((uint64) (MaxBlockNumber + 1))
47 |
48 |
49 | struct ColumnarScanDescData;
50 | typedef struct ColumnarScanDescData *ColumnarScanDesc;
51 |
52 |
53 | const TableAmRoutine * GetColumnarTableAmRoutine(void);
54 | extern void columnar_tableam_init(void);
55 | extern TableScanDesc columnar_beginscan_extended(Relation relation, Snapshot snapshot,
56 | int nkeys, ScanKey key,
57 | ParallelTableScanDesc parallel_scan,
58 | uint32 flags, Bitmapset *attr_needed,
59 | List *scanQual,
60 | ParallelColumnarScan parallelColumnarScan,
61 | bool returnVectorResult);
62 | extern IndexFetchTableData * columnar_index_fetch_begin_extended(Relation rel,
63 | Bitmapset *attr_neededs);
64 | extern int64 ColumnarScanChunkGroupsFiltered(ColumnarScanDesc columnarScanDesc);
65 | extern bool ColumnarSupportsIndexAM(char *indexAMName);
66 | extern bool IsColumnarTableAmTable(Oid relationId);
67 |
68 |
69 | #endif /* COLUMNAR_TABLEAM_H */
70 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/columnar_version_compat.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_version_compat.h
4 | *
5 | * Compatibility macros for writing code agnostic to PostgreSQL versions
6 | *
7 | * Copyright (c) Citus Data, Inc.
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 | #ifndef COLUMNAR_COMPAT_H
13 | #define COLUMNAR_COMPAT_H
14 |
15 | #include "pg_version_constants.h"
16 |
17 | #if PG_VERSION_NUM >= PG_VERSION_14
18 | #define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \
19 | ColumnarProcessUtility(a, b, c, d, e, f, g, h)
20 | #define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \
21 | PrevProcessUtilityHook(a, b, c, d, e, f, g, h)
22 | #define GetOldestNonRemovableTransactionId_compat(a, b) \
23 | GetOldestNonRemovableTransactionId(a)
24 | #define ExecSimpleRelationInsert_compat(a, b, c) \
25 | ExecSimpleRelationInsert(a, b, c)
26 | #define index_insert_compat(a, b, c, d, e, f, g, h) \
27 | index_insert(a, b, c, d, e, f, g, h)
28 | #else
29 | #define ColumnarProcessUtility_compat(a, b, c, d, e, f, g, h) \
30 | ColumnarProcessUtility(a, b, d, e, f, g, h)
31 | #define PrevProcessUtilityHook_compat(a, b, c, d, e, f, g, h) \
32 | PrevProcessUtilityHook(a, b, d, e, f, g, h)
33 | #define GetOldestNonRemovableTransactionId_compat(a, b) GetOldestXmin(a, b)
34 | #define ExecSimpleRelationInsert_compat(a, b, c) \
35 | ExecSimpleRelationInsert(b, c)
36 | #define index_insert_compat(a, b, c, d, e, f, g, h) \
37 | index_insert(a, b, c, d, e, f, h)
38 | #endif
39 |
40 | #define ACLCHECK_OBJECT_TABLE OBJECT_TABLE
41 |
42 | #define ExplainPropertyLong(qlabel, value, es) \
43 | ExplainPropertyInteger(qlabel, NULL, value, es)
44 |
45 | #if PG_VERSION_NUM < 130000
46 | #define detoast_attr(X) heap_tuple_untoast_attr(X)
47 | #endif
48 |
49 | #endif /* COLUMNAR_COMPAT_H */
50 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/columnar_write_state_row_mask.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_write_state_row_mask.h
4 | *
5 | * Declaration for Row Mask Entry
6 | *
7 | * Copyright (c) Hydra, Inc.
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 | #ifndef COLUMNAR_WRITE_STATE_ROW_MASK_H
13 | #define COLUMNAR_WRITE_STATE_ROW_MASK_H
14 |
15 | struct RowMaskWriteStateEntry
16 | {
17 | uint64 id;
18 | uint64 storageId;
19 | uint64 stripeId;
20 | uint32 chunkId;
21 | int64 startRowNumber;
22 | int64 endRowNumber;
23 | int32 deletedRows;
24 | bytea *mask;
25 | };
26 |
27 | #endif
--------------------------------------------------------------------------------
/columnar/src/include/columnar/vectorization/columnar_vector_execution.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_vector_execution.h
4 | *
5 | * Vectorization execution function
6 | *
7 | * Copyright (c) Hydra, Inc.
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 | #ifndef COLUMNAR_VECTOR_EXECUTION_H
13 | #define COLUMNAR_VECTOR_EXECUTION_H
14 |
15 | #include "executor/tuptable.h"
16 | #include "nodes/pg_list.h"
17 | #include "nodes/primnodes.h"
18 |
19 | extern bool CheckOpExprArgumentRules(List *args);
20 | extern bool GetVectorizedProcedureOid(Oid procedureOid, Oid *vectorizedProcedureOid);
21 | extern List * CreateVectorizedExprList(List *exprList);
22 | extern List * ConstructVectorizedQualList(TupleTableSlot *slot, List *vectorizedQual);
23 | extern bool * ExecuteVectorizedQual(TupleTableSlot *slot,
24 | List *vectorizedQualList,
25 | BoolExprType boolType,
26 | ExprContext *econtext);
27 |
28 | #endif
29 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/vectorization/columnar_vector_types.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_vector_types.h
4 | *
5 | * Structures used in vectorization execution
6 | *
7 | * Copyright (c) Hydra, Inc.
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 | #ifndef COLUMNAR_VECTOR_TYPES_H
13 | #define COLUMNAR_VECTOR_TYPES_H
14 |
15 | #include "postgres.h"
16 | #include "fmgr.h"
17 |
18 | #include "executor/tuptable.h"
19 | #include "nodes/bitmapset.h"
20 | #include "nodes/primnodes.h"
21 |
22 | /* DEFAULT_CHUNK_ROW_COUNT */
23 | #define COLUMNAR_VECTOR_COLUMN_SIZE 10000
24 |
25 | typedef struct VectorTupleTableSlot
26 | {
27 | /* TupleTableSlot structure */
28 | TupleTableSlot tts;
29 | /* How many tuples does this slot contain */
30 | uint32 dimension;
31 | /* Keep array to represent filtered tuples */
32 | bool keep[COLUMNAR_VECTOR_COLUMN_SIZE];
33 | /* Row Number */
34 | uint64 rowNumber[COLUMNAR_VECTOR_COLUMN_SIZE];
35 | } VectorTupleTableSlot;
36 |
37 | extern TupleTableSlot * CreateVectorTupleTableSlot(TupleDesc tupleDesc);
38 |
39 | typedef struct VectorColumn
40 | {
41 | uint32 dimension;
42 | uint16 columnTypeLen;
43 | bool columnIsVal;
44 | Datum *value;
45 | bool isnull[COLUMNAR_VECTOR_COLUMN_SIZE];
46 | uint64 *rowNumber;
47 | } VectorColumn;
48 |
49 | extern VectorColumn * BuildVectorColumn(int16 columnDimension,
50 | int16 columnTypeLen,
51 | bool columnIsVal,
52 | uint64 *rowNumber);
53 | extern void ExtractTupleFromVectorSlot(TupleTableSlot *out,
54 | VectorTupleTableSlot *vectorSlot,
55 | int32 index,
56 | List *attrNeededList);
57 | extern void WriteTupleToVectorSlot(TupleTableSlot *in,
58 | VectorTupleTableSlot *vectorSlot,
59 | int32 index);
60 | extern void CleanupVectorSlot(VectorTupleTableSlot *vectorSlot);
61 |
62 | typedef enum VectorQualType
63 | {
64 | VECTOR_QUAL_BOOL_EXPR,
65 | VECTOR_QUAL_EXPR
66 | } VectorQualTypeEnum;
67 |
68 |
69 | typedef enum VectorArgType
70 | {
71 | VECTOR_FN_ARG_CONSTANT,
72 | VECTOR_FN_ARG_VAR
73 | } VectorFnArgTypeEnum;
74 |
75 |
76 | typedef struct VectorFnArgument
77 | {
78 | VectorFnArgTypeEnum type;
79 | Datum arg;
80 | } VectorFnArgument;
81 |
82 |
83 | typedef struct VectorQual
84 | {
85 | VectorQualTypeEnum vectorQualType;
86 | union
87 | {
88 | struct
89 | {
90 | FmgrInfo *fmgrInfo;
91 | FunctionCallInfo fcInfo;
92 | VectorFnArgument *vectorFnArguments;
93 | } expr;
94 | struct
95 | {
96 | BoolExprType boolExprType;
97 | List *vectorQualExprList;
98 | } boolExpr;
99 | } u;
100 | } VectorQual;
101 |
102 | #endif
103 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/vectorization/nodes/columnar_aggregator_node.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * columnar_aggregator_node.h
4 | * Custom scan method for aggregation
5 | *
6 | * IDENTIFICATION
7 | * src/backend/columnar/vectorization/nodes/columnar_aggregator_node.c
8 | *
9 | *-------------------------------------------------------------------------
10 | */
11 |
12 |
13 | #ifndef COLUMNAR_AGGEREGATOR_NODE_H
14 | #define COLUMNAR_AGGEREGATOR_NODE_H
15 |
16 | #include "postgres.h"
17 |
18 | #include "nodes/execnodes.h"
19 |
20 | typedef struct VectorAggState
21 | {
22 | CustomScanState css;
23 | AggState *aggstate;
24 | } VectorAggState;
25 |
26 | extern CustomScan *columnar_create_aggregator_node(void);
27 | extern void columnar_register_aggregator_node(void);
28 |
29 | #endif
30 |
--------------------------------------------------------------------------------
/columnar/src/include/columnar/vectorization/types/numeric.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * numeric.h
4 | *
5 | * IDENTIFICATION
6 | * src/backend/columnar/vectorization/types/numeric.c
7 | *
8 | *-------------------------------------------------------------------------
9 | */
10 |
11 | #ifndef VECTORIZATION_TYPE_INT128_H
12 | #define VECTORIZATION_TYPE_INT128_H
13 |
14 | #include "postgres.h"
15 |
16 | #include "utils/numeric.h"
17 | extern Numeric int128_to_numeric(int128 val);
18 |
19 | #endif
--------------------------------------------------------------------------------
/columnar/src/include/pg_version_constants.h:
--------------------------------------------------------------------------------
1 | /*-------------------------------------------------------------------------
2 | *
3 | * pg_version_constants.h
4 | * pg version related constants.
5 | *
6 | * Copyright (c) Citus Data, Inc.
7 | *
8 | *-------------------------------------------------------------------------
9 | */
10 |
11 | #ifndef PG_VERSION_CONSTANTS
12 | #define PG_VERSION_CONSTANTS
13 |
14 | #define PG_VERSION_12 120000
15 | #define PG_VERSION_13 130000
16 | #define PG_VERSION_14 140000
17 | #define PG_VERSION_15 150000
18 | #define PG_VERSION_16 160000
19 |
20 | #endif /* PG_VERSION_CONSTANTS */
21 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/.gitignore:
--------------------------------------------------------------------------------
1 | # Local binaries
2 | /pg_regress
3 |
4 | # Generated subdirectories
5 | /tmp_check/
6 | /results/
7 | /log/
8 |
9 | # Regression test output
10 | /regression.diffs
11 | /regression.output
12 | /regression.out
13 |
14 | # python
15 | *.pyc
16 |
17 | # core dumps
18 | core
19 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/Makefile:
--------------------------------------------------------------------------------
1 | # Makefile for tests of the Citus extension
2 |
3 | citus_subdir = src/test/regress
4 | citus_top_builddir = ../../..
5 |
6 | include $(citus_top_builddir)/Makefile.global
7 |
8 | PG_VERSION_NUM := $(shell cat `$(PG_CONFIG) --includedir-server`/pg_config*.h \
9 | | perl -ne 'print $1 and exit if /PG_VERSION_NUM\s+(\d+)/')
10 |
11 | # Test input and expected files. These are created by pg_regress itself, so we
12 | # don't have a rule to create them. We do need rules to clean them however.
13 | input_files := $(patsubst $(citus_abs_srcdir)/input/%.source,sql/%.sql, $(wildcard $(citus_abs_srcdir)/input/*.source))
14 | output_files := $(patsubst $(citus_abs_srcdir)/output/%.source,expected/%.out, $(wildcard $(citus_abs_srcdir)/output/*.source))
15 |
16 | check-all: check-regression-columnar
17 |
18 | check-regression-columnar:
19 | ifeq ($(shell test $(PG_VERSION_NUM) -gt 149999; echo $?),0)
20 | # postgres 15 removed the whole concept of input/output files
21 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < input/columnar_copyto.source > sql/columnar_copyto.sql
22 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < input/columnar_data_types.source > sql/columnar_data_types.sql
23 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < input/columnar_load.source > sql/columnar_load.sql
24 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < output/columnar_copyto.source > expected/columnar_copyto.out
25 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < output/columnar_data_types.source > expected/columnar_data_types.out
26 | sed "s#@abs_srcdir@#$(PWD)/src/test/regress#g" < output/columnar_load.source > expected/columnar_load.out
27 | endif
28 | TEST_DIR=$(PWD) $(pg_regress_check) \
29 | --temp-config columnar_regression.conf \
30 | --load-extension=columnar \
31 | --schedule=$(citus_abs_srcdir)/columnar_schedule
32 |
33 | clean-regression:
34 | rm -fr $(citus_abs_srcdir)/tmp_check
35 | rm -fr $(citus_abs_srcdir)/log
36 | rm -fr $(citus_abs_srcdir)/results
37 | rm -f $(citus_abs_srcdir)/regression.diffs
38 | rm -f $(citus_abs_srcdir)/regression.out
39 | rm -f $(citus_abs_srcdir)/$(output_files)
40 | rm -fr $(citus_abs_srcdir)/$(input_files)
41 |
42 | clean: clean-regression
43 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/columnar_regression.conf:
--------------------------------------------------------------------------------
1 | # Columnar storage engine configuration
2 |
3 | shared_preload_libraries = 'columnar.so'
4 | log_temp_files = -1
--------------------------------------------------------------------------------
/columnar/src/test/regress/columnar_schedule:
--------------------------------------------------------------------------------
1 | test: columnar_test_helpers
2 |
3 | test: columnar_create
4 | test: columnar_load
5 | test: columnar_query columnar_first_row_number
6 | test: columnar_analyze
7 | test: columnar_data_types
8 | test: columnar_drop
9 | test: columnar_indexes
10 | test: columnar_fallback_scan columnar_paths
11 | #test: columnar_partitioning
12 | test: columnar_permissions
13 | test: columnar_empty
14 | test: columnar_insert
15 | test: columnar_update_delete
16 | test: columnar_cursor
17 | test: columnar_copyto
18 | test: columnar_alter
19 | test: columnar_alter_set_type
20 | test: columnar_lz4 columnar_zstd
21 | test: columnar_rollback
22 | test: columnar_truncate
23 | test: columnar_vacuum
24 | test: columnar_vacuum_udf
25 | test: columnar_clean
26 | test: columnar_types_without_comparison
27 | #test: columnar_chunk_filtering
28 | test: columnar_join
29 | test: columnar_trigger
30 | test: columnar_tableoptions
31 | test: columnar_recursive
32 | test: columnar_transactions
33 | test: columnar_matview
34 | #test: columnar_memory
35 | test: columnar_alter_table_set_access_method
36 | test: columnar_cache
37 | test: columnar_aggregates
38 | test: columnar_upsert
39 | test: columnar_customindex
40 | test: columnar_vectorization
41 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/data/array_types.csv:
--------------------------------------------------------------------------------
1 | "{1,2,3}","{1,2,3}","{a,b,c}"
2 | {},{},{}
3 | "{-2147483648,2147483647}","{-9223372036854775808,9223372036854775807}","{""""}"
4 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/data/contestants.1.csv:
--------------------------------------------------------------------------------
1 | a,1990-01-10,2090,97.1,XA ,{a}
2 | b,1990-11-01,2203,98.1,XA ,"{a,b}"
3 | c,1988-11-01,2907,99.4,XB ,"{w,y}"
4 | d,1985-05-05,2314,98.3,XB ,{}
5 | e,1995-05-05,2236,98.2,XC ,{a}
6 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/data/contestants.2.csv:
--------------------------------------------------------------------------------
1 | f,1983-04-02,3090,99.6,XD ,"{a,b,c,y}"
2 | g,1991-12-13,1803,85.1,XD ,"{a,c}"
3 | h,1987-10-26,2112,95.4,XD ,"{w,a}"
4 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/data/datetime_types.csv:
--------------------------------------------------------------------------------
1 | 2000-01-02 04:05:06,1999-01-08 14:05:06+02,2000-01-02,04:05:06,04:00:00
2 | 1970-01-01 00:00:00,infinity,-infinity,00:00:00,00:00:00
3 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/data/enum_and_composite_types.csv:
--------------------------------------------------------------------------------
1 | a,"(2,b)"
2 | b,"(3,c)"
3 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/data/null_values.csv:
--------------------------------------------------------------------------------
1 | ,{NULL},"(,)"
2 | ,,
3 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/data/other_types.csv:
--------------------------------------------------------------------------------
1 | f,\xdeadbeef,$1.00,192.168.1.2,10101,a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11,"{""key"": ""value""}"
2 | t,\xcdb0,$1.50,127.0.0.1,"",a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11,[]
3 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/data/range_types.csv:
--------------------------------------------------------------------------------
1 | "[1,3)","[1,3)","[1,3)","[""2000-01-02 00:30:00"",""2010-02-03 12:30:00"")"
2 | empty,"[1,)","(,)",empty
3 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/.gitignore:
--------------------------------------------------------------------------------
1 | /columnar_copyto.out
2 | /columnar_data_types.out
3 | /columnar_load.out
4 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_alter_set_type.out:
--------------------------------------------------------------------------------
1 | --
2 | -- Testing ALTER TABLE on columnar tables.
3 | --
4 | CREATE TABLE test_alter_table (a int, b int, c text) USING columnar;
5 | WITH sample_data AS (VALUES
6 | (1, 2, '3'),
7 | (4, 5, '6')
8 | )
9 | INSERT INTO test_alter_table SELECT * FROM sample_data;
10 | WITH sample_data AS (VALUES
11 | (5, 9, '11'),
12 | (12, 83, '93')
13 | )
14 | INSERT INTO test_alter_table SELECT * FROM sample_data;
15 | ALTER TABLE test_alter_table ALTER COLUMN a TYPE jsonb USING row_to_json(row(a));
16 | SELECT * FROM test_alter_table ORDER BY a;
17 | a | b | c
18 | ------------+----+----
19 | {"f1": 1} | 2 | 3
20 | {"f1": 4} | 5 | 6
21 | {"f1": 5} | 9 | 11
22 | {"f1": 12} | 83 | 93
23 | (4 rows)
24 |
25 | ALTER TABLE test_alter_table ALTER COLUMN c TYPE int USING c::integer;
26 | SELECT sum(c) FROM test_alter_table;
27 | sum
28 | -----
29 | 113
30 | (1 row)
31 |
32 | ALTER TABLE test_alter_table ALTER COLUMN b TYPE bigint;
33 | SELECT * FROM test_alter_table ORDER BY a;
34 | a | b | c
35 | ------------+----+----
36 | {"f1": 1} | 2 | 3
37 | {"f1": 4} | 5 | 6
38 | {"f1": 5} | 9 | 11
39 | {"f1": 12} | 83 | 93
40 | (4 rows)
41 |
42 | ALTER TABLE test_alter_table ALTER COLUMN b TYPE float USING (b::float + 0.5);
43 | SELECT * FROM test_alter_table ORDER BY a;
44 | a | b | c
45 | ------------+------+----
46 | {"f1": 1} | 2.5 | 3
47 | {"f1": 4} | 5.5 | 6
48 | {"f1": 5} | 9.5 | 11
49 | {"f1": 12} | 83.5 | 93
50 | (4 rows)
51 |
52 | DROP TABLE test_alter_table;
53 | -- Make sure that the correct table options are used when rewriting the table.
54 | -- This is reflected by the VACUUM VERBOSE output right after a rewrite showing
55 | -- that all chunks are compressed with the configured compression algorithm
56 | CREATE TABLE test(i int) USING columnar;
57 | SELECT columnar.alter_columnar_table_set('test', compression => 'lz4');
58 | alter_columnar_table_set
59 | --------------------------
60 |
61 | (1 row)
62 |
63 | INSERT INTO test VALUES(1);
64 | VACUUM VERBOSE test;
65 | INFO: statistics for "test":
66 | storage id: 10000000144
67 | total file size: 24576, total data size: 6
68 | compression rate: 0.83x
69 | total row count: 1, stripe count: 1, average rows per stripe: 1
70 | chunk count: 1, containing data for dropped columns: 0, lz4 compressed: 1
71 |
72 | ALTER TABLE test ALTER COLUMN i TYPE int8;
73 | VACUUM VERBOSE test;
74 | INFO: statistics for "test":
75 | storage id: 10000000145
76 | total file size: 24576, total data size: 10
77 | compression rate: 0.90x
78 | total row count: 1, stripe count: 1, average rows per stripe: 1
79 | chunk count: 1, containing data for dropped columns: 0, lz4 compressed: 1
80 |
81 | DROP TABLE test;
82 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_analyze.out:
--------------------------------------------------------------------------------
1 | --
2 | -- Test the ANALYZE command for columnar tables.
3 | --
4 | -- ANALYZE uncompressed table
5 | ANALYZE contestant;
6 | SELECT count(*) FROM pg_stats WHERE tablename='contestant';
7 | count
8 | -------
9 | 6
10 | (1 row)
11 |
12 | -- ANALYZE compressed table
13 | ANALYZE contestant_compressed;
14 | SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed';
15 | count
16 | -------
17 | 6
18 | (1 row)
19 |
20 | -- ANALYZE a table with lots of data to trigget qsort in analyze.c
21 | CREATE TABLE test_analyze(a int, b text, c char) USING columnar;
22 | INSERT INTO test_analyze SELECT floor(i / 1000), floor(i / 10)::text, 4 FROM generate_series(1, 100000) i;
23 | INSERT INTO test_analyze SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 110000) i;
24 | ANALYZE test_analyze;
25 | DROP TABLE test_analyze;
26 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_clean.out:
--------------------------------------------------------------------------------
1 | DROP TABLE test_null_values;
2 | DROP TABLE test_other_types;
3 | DROP TABLE test_range_types;
4 | DROP TABLE test_enum_and_composite_types;
5 | DROP TYPE composite_type;
6 | DROP TYPE enum_type;
7 | DROP TABLE test_datetime_types;
8 | DROP TABLE test_array_types;
9 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_customindex.out:
--------------------------------------------------------------------------------
1 | --
2 | -- Test custom index: Test shows only that we are exchanging Index scan with CustomIndexScan while
3 | -- all other information is still the same. The feature is not visible through EXPLAIN because output
4 | -- will be same in both cases, but difference is noticeable how the columnar storage engine is read.
5 | -- With index scan ALL columns are requested from storage (this can be significant overhead in performance)
6 | -- while with CustomIndexScan we will request only columns that are needed - that are going to be used as output.
7 | -- Test also shows that we are not using CustomIndexScan with heap tables and only on columnar tables.
8 | --
9 | SET columnar.enable_columnar_index_scan TO TRUE;
10 | CREATE TABLE t(a INT PRIMARY KEY, b INT, c TEXT) USING columnar;
11 | CREATE INDEX t_idx ON t USING btree(b);
12 | CREATE TABLE t_heap(a INT PRIMARY KEY, b INT, c TEXT);
13 | CREATE INDEX t_idx_heap ON t_heap USING btree(b);
14 | INSERT INTO t SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g;
15 | INSERT INTO t_heap SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g;
16 | -- make sure that we test index scan
17 | set columnar.enable_custom_scan TO 'off';
18 | set enable_seqscan TO off;
19 | set seq_page_cost TO 10000000;
20 | EXPLAIN (VERBOSE) SELECT a FROM t WHERE b > 18 ORDER BY b LIMIT 10;
21 | QUERY PLAN
22 | ---------------------------------------------------------------------------------
23 | Limit (cost=0.30..0.56 rows=10 width=8)
24 | Output: a, b
25 | -> Custom Scan (ColumnarIndexScan) (cost=0.30..2586.30 rows=100000 width=8)
26 | Output: a, b
27 | ColumnarIndexScan using : t_idx
28 | Index Cond: (t.b > 18)
29 | (6 rows)
30 |
31 | EXPLAIN (VERBOSE) SELECT a FROM t_heap WHERE b > 18 ORDER BY b LIMIT 10;
32 | QUERY PLAN
33 | ---------------------------------------------------------------------------------------------
34 | Limit (cost=0.29..1.51 rows=10 width=8)
35 | Output: a, b
36 | -> Index Scan using t_idx_heap on public.t_heap (cost=0.29..9305.99 rows=76440 width=8)
37 | Output: a, b
38 | Index Cond: (t_heap.b > 18)
39 | (5 rows)
40 |
41 | DROP TABLE t;
42 | DROP TABLE t_heap;
43 | SET columnar.enable_custom_index_scan TO default;
44 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_customindex_1.out:
--------------------------------------------------------------------------------
1 | --
2 | -- Test custom index: Test shows only that we are exchanging Index scan with CustomIndexScan while
3 | -- all other information is still the same. The feature is not visible through EXPLAIN because output
4 | -- will be same in both cases, but difference is noticeable how the columnar storage engine is read.
5 | -- With index scan ALL columns are requested from storage (this can be significant overhead in performance)
6 | -- while with CustomIndexScan we will request only columns that are needed - that are going to be used as output.
7 | -- Test also shows that we are not using CustomIndexScan with heap tables and only on columnar tables.
8 | --
9 | SET columnar.enable_columnar_index_scan TO TRUE;
10 | CREATE TABLE t(a INT PRIMARY KEY, b INT, c TEXT) USING columnar;
11 | CREATE INDEX t_idx ON t USING btree(b);
12 | CREATE TABLE t_heap(a INT PRIMARY KEY, b INT, c TEXT);
13 | CREATE INDEX t_idx_heap ON t_heap USING btree(b);
14 | INSERT INTO t SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g;
15 | INSERT INTO t_heap SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g;
16 | -- make sure that we test index scan
17 | set columnar.enable_custom_scan TO 'off';
18 | set enable_seqscan TO off;
19 | set seq_page_cost TO 10000000;
20 | EXPLAIN (VERBOSE) SELECT a FROM t WHERE b > 18 ORDER BY b LIMIT 10;
21 | QUERY PLAN
22 | ------------------------------------------------------------------------------------
23 | Limit (cost=0.30..0.56 rows=10 width=8)
24 | Output: a, b
25 | -> Index Scan using t_idx on public.t (cost=0.30..2586.30 rows=100000 width=8)
26 | Output: a, b
27 | Index Cond: (t.b > 18)
28 | (5 rows)
29 |
30 | EXPLAIN (VERBOSE) SELECT a FROM t_heap WHERE b > 18 ORDER BY b LIMIT 10;
31 | QUERY PLAN
32 | ---------------------------------------------------------------------------------------------
33 | Limit (cost=0.29..1.51 rows=10 width=8)
34 | Output: a, b
35 | -> Index Scan using t_idx_heap on public.t_heap (cost=0.29..9305.99 rows=76440 width=8)
36 | Output: a, b
37 | Index Cond: (t_heap.b > 18)
38 | (5 rows)
39 |
40 | DROP TABLE t;
41 | DROP TABLE t_heap;
42 | SET columnar.enable_custom_index_scan TO default;
43 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_drop.out:
--------------------------------------------------------------------------------
1 | --
2 | -- Tests the different DROP commands for columnar tables.
3 | --
4 | -- DROP TABL
5 | -- DROP SCHEMA
6 | -- DROP EXTENSION
7 | -- DROP DATABASE
8 | --
9 | -- Note that travis does not create
10 | -- citus extension in default database (postgres). This has caused
11 | -- different behavior between travis tests and local tests. Thus
12 | -- 'postgres' directory is excluded from comparison to have the same result.
13 | -- store postgres database oid
14 | SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
15 | SELECT count(distinct storage_id) AS columnar_stripes_before_drop FROM columnar.stripe \gset
16 | -- DROP columnar tables
17 | DROP TABLE contestant;
18 | DROP TABLE contestant_compressed;
19 | -- make sure DROP deletes metadata
20 | SELECT :columnar_stripes_before_drop - count(distinct storage_id) FROM columnar.stripe;
21 | ?column?
22 | ----------
23 | 2
24 | (1 row)
25 |
26 | -- Create a columnar table under a schema and drop it.
27 | CREATE SCHEMA test_schema;
28 | CREATE TABLE test_schema.test_table(data int) USING columnar;
29 | INSERT INTO test_schema.test_table VALUES (1);
30 | SELECT count(*) AS columnar_stripes_before_drop FROM columnar.stripe \gset
31 | DROP SCHEMA test_schema CASCADE;
32 | NOTICE: drop cascades to table test_schema.test_table
33 | SELECT :columnar_stripes_before_drop - count(distinct storage_id) FROM columnar.stripe;
34 | ?column?
35 | ----------
36 | 1
37 | (1 row)
38 |
39 | SELECT current_database() datname \gset
40 | CREATE DATABASE db_to_drop;
41 | \c db_to_drop
42 | CREATE EXTENSION columnar;
43 | SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
44 | CREATE TABLE test_table(data int) USING columnar;
45 | DROP EXTENSION columnar CASCADE;
46 | NOTICE: drop cascades to table test_table
47 | -- test database drop
48 | CREATE EXTENSION columnar;
49 | SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
50 | CREATE TABLE test_table(data int) USING columnar;
51 | \c :datname
52 | DROP DATABASE db_to_drop;
53 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_fallback_scan.out:
--------------------------------------------------------------------------------
1 | --
2 | -- columnar_fallback_scan.sql
3 | --
4 | -- Test columnar.enable_custom_scan = false, which will use an
5 | -- ordinary sequential scan. It won't benefit from projection pushdown
6 | -- or qual pushdown, but it should return the correct results.
7 | --
8 | set columnar.enable_custom_scan = false;
9 | create table fallback_scan(i int) using columnar;
10 | -- large enough to test parallel_workers > 1
11 | select columnar.alter_columnar_table_set('fallback_scan', compression => 'none');
12 | alter_columnar_table_set
13 | --------------------------
14 |
15 | (1 row)
16 |
17 | insert into fallback_scan select generate_series(1,150000);
18 | vacuum analyze fallback_scan;
19 | select count(*), min(i), max(i), avg(i) from fallback_scan;
20 | count | min | max | avg
21 | --------+-----+--------+--------------------
22 | 150000 | 1 | 150000 | 75000.500000000000
23 | (1 row)
24 |
25 | --
26 | -- Negative test: try to force a parallel plan with at least two
27 | -- workers, but columnar should reject it and use a non-parallel scan.
28 | --
29 | set min_parallel_table_scan_size = 1;
30 | set parallel_tuple_cost = 0;
31 | set max_parallel_workers = 4;
32 | set max_parallel_workers_per_gather = 4;
33 | explain (costs off) select count(*), min(i), max(i), avg(i) from fallback_scan;
34 | QUERY PLAN
35 | ---------------------------------
36 | Aggregate
37 | -> Seq Scan on fallback_scan
38 | (2 rows)
39 |
40 | select count(*), min(i), max(i), avg(i) from fallback_scan;
41 | count | min | max | avg
42 | --------+-----+--------+--------------------
43 | 150000 | 1 | 150000 | 75000.500000000000
44 | (1 row)
45 |
46 | set min_parallel_table_scan_size to default;
47 | set parallel_tuple_cost to default;
48 | set max_parallel_workers to default;
49 | set max_parallel_workers_per_gather to default;
50 | set columnar.enable_custom_scan to default;
51 | drop table fallback_scan;
52 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_first_row_number.out:
--------------------------------------------------------------------------------
1 | CREATE SCHEMA columnar_first_row_number;
2 | SET search_path tO columnar_first_row_number;
3 | CREATE TABLE col_table_1 (a int) USING columnar;
4 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 10) i;
5 | BEGIN;
6 | -- we don't use same first_row_number even if the xact is rollback'ed
7 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 11) i;
8 | ROLLBACK;
9 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 12) i;
10 | SELECT columnar.alter_columnar_table_set('col_table_1', stripe_row_limit => 1000);
11 | alter_columnar_table_set
12 | --------------------------
13 |
14 | (1 row)
15 |
16 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 2350) i;
17 | SELECT row_count, first_row_number FROM columnar.stripe a
18 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass)
19 | ORDER BY stripe_num;
20 | row_count | first_row_number
21 | -----------+------------------
22 | 10 | 1
23 | 12 | 300001
24 | 1000 | 450001
25 | 1000 | 451001
26 | 350 | 452001
27 | (5 rows)
28 |
29 | VACUUM FULL col_table_1;
30 | -- show that we properly update first_row_number after VACUUM FULL
31 | SELECT row_count, first_row_number FROM columnar.stripe a
32 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass)
33 | ORDER BY stripe_num;
34 | row_count | first_row_number
35 | -----------+------------------
36 | 1000 | 1
37 | 1000 | 1001
38 | 372 | 2001
39 | (3 rows)
40 |
41 | TRUNCATE col_table_1;
42 | BEGIN;
43 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 16) i;
44 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 16) i;
45 | COMMIT;
46 | -- show that we start with first_row_number=1 after TRUNCATE
47 | SELECT row_count, first_row_number FROM columnar.stripe a
48 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass)
49 | ORDER BY stripe_num;
50 | row_count | first_row_number
51 | -----------+------------------
52 | 32 | 1
53 | (1 row)
54 |
55 | SET client_min_messages TO ERROR;
56 | DROP SCHEMA columnar_first_row_number CASCADE;
57 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_join.out:
--------------------------------------------------------------------------------
1 | CREATE SCHEMA am_columnar_join;
2 | SET search_path TO am_columnar_join;
3 | CREATE TABLE users (id int, name text) USING columnar;
4 | INSERT INTO users SELECT a, 'name' || a FROM generate_series(0,30-1) AS a;
5 | CREATE TABLE things (id int, user_id int, name text) USING columnar;
6 | INSERT INTO things SELECT a, a % 30, 'thing' || a FROM generate_series(1,300) AS a;
7 | -- force the nested loop to rescan the table
8 | SET enable_material TO off;
9 | SET enable_hashjoin TO off;
10 | SET enable_mergejoin TO off;
11 | SELECT count(*)
12 | FROM users
13 | JOIN things ON (users.id = things.user_id)
14 | WHERE things.id > 290;
15 | count
16 | -------
17 | 10
18 | (1 row)
19 |
20 | -- verify the join uses a nested loop to trigger the rescan behaviour
21 | EXPLAIN (COSTS OFF)
22 | SELECT count(*)
23 | FROM users
24 | JOIN things ON (users.id = things.user_id)
25 | WHERE things.id > 299990;
26 | QUERY PLAN
27 | --------------------------------------------------------------------------------------
28 | Aggregate
29 | -> Nested Loop
30 | -> Custom Scan (ColumnarScan) on users
31 | Columnar Projected Columns: id
32 | -> Custom Scan (ColumnarScan) on things
33 | Filter: ((id > 299990) AND (users.id = user_id))
34 | Columnar Projected Columns: id, user_id
35 | Columnar Chunk Group Filters: ((id > 299990) AND (users.id = user_id))
36 | (8 rows)
37 |
38 | EXPLAIN (COSTS OFF)
39 | SELECT u1.id, u2.id, COUNT(u2.*)
40 | FROM users u1
41 | JOIN users u2 ON (u1.id::text = u2.name)
42 | WHERE u2.id > 299990
43 | GROUP BY u1.id, u2.id;
44 | QUERY PLAN
45 | ----------------------------------------------------------------------------------------------
46 | GroupAggregate
47 | Group Key: u1.id, u2.id
48 | -> Sort
49 | Sort Key: u1.id, u2.id
50 | -> Nested Loop
51 | -> Custom Scan (ColumnarScan) on users u1
52 | Columnar Projected Columns: id
53 | -> Custom Scan (ColumnarScan) on users u2
54 | Filter: ((id > 299990) AND ((u1.id)::text = name))
55 | Columnar Projected Columns: id, name
56 | Columnar Chunk Group Filters: ((id > 299990) AND ((u1.id)::text = name))
57 | (11 rows)
58 |
59 | SET client_min_messages TO warning;
60 | DROP SCHEMA am_columnar_join CASCADE;
61 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_lz4.out:
--------------------------------------------------------------------------------
1 | SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset
2 | \if :lz4_supported
3 | \else
4 | \q
5 | \endif
6 | CREATE SCHEMA am_alz4;
7 | SET search_path TO am_alz4;
8 | SET columnar.compression TO 'lz4';
9 | CREATE TABLE test_lz4 (a int, b text, c int) USING columnar;
10 | INSERT INTO test_lz4 SELECT floor(i / 1000), floor(i / 10)::text, 4 FROM generate_series(1, 10000) i;
11 | SELECT count(*) FROM test_lz4;
12 | count
13 | -------
14 | 10000
15 | (1 row)
16 |
17 | INSERT INTO test_lz4 SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 11000) i;
18 | SELECT count(*) FROM test_lz4;
19 | count
20 | -------
21 | 20001
22 | (1 row)
23 |
24 | SELECT pg_relation_size('test_lz4') AS size_lz4 \gset
25 | SELECT DISTINCT * FROM test_lz4 ORDER BY a, b, c LIMIT 5;
26 | a | b | c
27 | ---+----+---
28 | 0 | 0 | 4
29 | 0 | 1 | 4
30 | 0 | 10 | 4
31 | 0 | 11 | 4
32 | 0 | 12 | 4
33 | (5 rows)
34 |
35 | -- compare compression rate to pglz
36 | SET columnar.compression TO 'pglz';
37 | CREATE TABLE test_pglz (LIKE test_lz4) USING columnar;
38 | INSERT INTO test_pglz SELECT * FROM test_lz4;
39 | SELECT pg_relation_size('test_pglz') AS size_pglz \gset
40 | -- verify that pglz & lz4 resulted in different compression ratios
41 | SELECT :size_pglz <> :size_lz4;
42 | ?column?
43 | ----------
44 | t
45 | (1 row)
46 |
47 | -- Other operations
48 | VACUUM FULL test_lz4;
49 | ANALYZE test_lz4;
50 | SELECT count(DISTINCT test_lz4.*) FROM test_lz4;
51 | count
52 | -------
53 | 6002
54 | (1 row)
55 |
56 | TRUNCATE test_lz4;
57 | SELECT count(DISTINCT test_lz4.*) FROM test_lz4;
58 | count
59 | -------
60 | 0
61 | (1 row)
62 |
63 | SET client_min_messages TO WARNING;
64 | DROP SCHEMA am_alz4 CASCADE;
65 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_permissions.out:
--------------------------------------------------------------------------------
1 | select current_user \gset
2 | create user columnar_user;
3 | grant all on schema public to columnar_user;
4 | \c - columnar_user
5 | create table columnar_permissions(i int) using columnar;
6 | insert into columnar_permissions values(1);
7 | alter table columnar_permissions add column j int;
8 | insert into columnar_permissions values(2,20);
9 | vacuum columnar_permissions;
10 | truncate columnar_permissions;
11 | drop table columnar_permissions;
12 | \c - :current_user
13 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_upsert.out:
--------------------------------------------------------------------------------
1 | CREATE TABLE upsert_test (
2 | i INT,
3 | t TEXT
4 | ) USING columnar;
5 | CREATE UNIQUE INDEX ON upsert_test (t);
6 | INSERT INTO upsert_test (i, t) VALUES (1, 'hello');
7 | INSERT INTO upsert_test (i, t) VALUES (2, 'world');
8 | -- should work, then roll back
9 | BEGIN;
10 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'foo';
11 | SELECT * FROM upsert_test;
12 | i | t
13 | ---+-------
14 | 2 | world
15 | 1 | foo
16 | (2 rows)
17 |
18 | ROLLBACK;
19 | SELECT * FROM upsert_test;
20 | i | t
21 | ---+-------
22 | 1 | hello
23 | 2 | world
24 | (2 rows)
25 |
26 | -- should take affect immediately
27 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'bar';
28 | SELECT * FROM upsert_test;
29 | i | t
30 | ---+-------
31 | 2 | world
32 | 1 | bar
33 | (2 rows)
34 |
35 | -- should work as expected
36 | BEGIN;
37 | INSERT INTO upsert_test (t) VALUES ( 'world' ) ON CONFLICT (t) DO UPDATE SET t = 'foo';
38 | SELECT * FROM upsert_test;
39 | i | t
40 | ---+-----
41 | 1 | bar
42 | 2 | foo
43 | (2 rows)
44 |
45 | COMMIT;
46 | -- should also work as expected, a select can mask a bug
47 | DELETE FROM upsert_test;
48 | BEGIN;
49 | INSERT INTO upsert_test (i, t) VALUES (1, 'hello');
50 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'bar';
51 | COMMIT;
52 | SELECT * FROM upsert_test;
53 | i | t
54 | ---+-----
55 | 1 | bar
56 | (1 row)
57 |
58 | DROP TABLE upsert_test;
59 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_vectorization.out:
--------------------------------------------------------------------------------
1 | CREATE TABLE t (id int, ts timestamp) USING columnar;
2 | INSERT INTO t SELECT id, now() + id * '1 day'::interval FROM generate_series(1, 100000) id;
3 | EXPLAIN (costs off) SELECT * FROM t WHERE ts between '2026-01-01'::timestamp and '2026-02-01'::timestamp;
4 | QUERY PLAN
5 | -----------------------------------------------------------------------------------------------------------------------------------------------------------------------
6 | Custom Scan (ColumnarScan) on t
7 | Columnar Projected Columns: id, ts
8 | Columnar Chunk Group Filters: ((ts >= 'Thu Jan 01 00:00:00 2026'::timestamp without time zone) AND (ts <= 'Sun Feb 01 00:00:00 2026'::timestamp without time zone))
9 | Columnar Vectorized Filter: ((ts >= 'Thu Jan 01 00:00:00 2026'::timestamp without time zone) AND (ts <= 'Sun Feb 01 00:00:00 2026'::timestamp without time zone))
10 | (4 rows)
11 |
12 | DROP TABLE t;
13 | CREATE TABLE t (id int, ts timestamptz) USING columnar;
14 | INSERT INTO t SELECT id, now() + id * '1 day'::interval FROM generate_series(1, 100000) id;
15 | EXPLAIN (costs off) SELECT * FROM t WHERE ts between '2026-01-01'::timestamptz and '2026-02-01'::timestamptz;
16 | QUERY PLAN
17 | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------
18 | Custom Scan (ColumnarScan) on t
19 | Columnar Projected Columns: id, ts
20 | Columnar Chunk Group Filters: ((ts >= 'Thu Jan 01 00:00:00 2026 PST'::timestamp with time zone) AND (ts <= 'Sun Feb 01 00:00:00 2026 PST'::timestamp with time zone))
21 | Columnar Vectorized Filter: ((ts >= 'Thu Jan 01 00:00:00 2026 PST'::timestamp with time zone) AND (ts <= 'Sun Feb 01 00:00:00 2026 PST'::timestamp with time zone))
22 | (4 rows)
23 |
24 | DROP TABLE t;
25 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/expected/columnar_zstd.out:
--------------------------------------------------------------------------------
1 | SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset
2 | \if :zstd_supported
3 | \else
4 | \q
5 | \endif
6 | CREATE SCHEMA am_zstd;
7 | SET search_path TO am_zstd;
8 | SET columnar.compression TO 'zstd';
9 | CREATE TABLE test_zstd (a int, b text, c int) USING columnar;
10 | INSERT INTO test_zstd SELECT i % 1000, (i % 10)::text, 4 FROM generate_series(1, 10000) i;
11 | SELECT count(*) FROM test_zstd;
12 | count
13 | -------
14 | 10000
15 | (1 row)
16 |
17 | INSERT INTO test_zstd SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 11000) i;
18 | SELECT count(*) FROM test_zstd;
19 | count
20 | -------
21 | 20001
22 | (1 row)
23 |
24 | CREATE TABLE test_none (LIKE test_zstd) USING columnar;
25 | INSERT INTO test_none SELECT * FROM test_zstd;
26 | SELECT DISTINCT * FROM test_zstd ORDER BY a, b, c LIMIT 5;
27 | a | b | c
28 | ---+---+---
29 | 0 | 0 | 4
30 | 1 | 1 | 4
31 | 2 | 2 | 4
32 | 3 | 3 | 4
33 | 4 | 4 | 4
34 | (5 rows)
35 |
36 | VACUUM FULL test_zstd;
37 | -- Hydra: we need to get `data_length` from stripe metadata information to compare compression
38 | -- rather than calling pg_relation_size function
39 | SELECT data_length AS size_comp_level_default FROM columnar.stripe WHERE storage_id = (
40 | SELECT storage_id from columnar_test_helpers.columnar_storage_info('test_zstd')) \gset
41 | -- change compression level
42 | SELECT columnar.alter_columnar_table_set('test_zstd', compression_level => 19);
43 | alter_columnar_table_set
44 | --------------------------
45 |
46 | (1 row)
47 |
48 | VACUUM FULL test_zstd;
49 | SELECT data_length AS size_comp_level_19 FROM columnar.stripe WHERE storage_id = (
50 | SELECT storage_id from columnar_test_helpers.columnar_storage_info('test_zstd')) \gset
51 | -- verify that higher compression level compressed better
52 | SELECT :size_comp_level_default > :size_comp_level_19 AS size_changed;
53 | size_changed
54 | --------------
55 | t
56 | (1 row)
57 |
58 | -- compare compression rate to pglz
59 | SET columnar.compression TO 'pglz';
60 | CREATE TABLE test_pglz (LIKE test_zstd) USING columnar;
61 | INSERT INTO test_pglz SELECT * FROM test_zstd;
62 | SELECT pg_relation_size('test_pglz') AS size_pglz \gset
63 | -- verify that zstd compressed better than pglz
64 | SELECT :size_pglz > :size_comp_level_default;
65 | ?column?
66 | ----------
67 | t
68 | (1 row)
69 |
70 | -- Other operations
71 | ANALYZE test_zstd;
72 | SELECT count(DISTINCT test_zstd.*) FROM test_zstd;
73 | count
74 | -------
75 | 6001
76 | (1 row)
77 |
78 | TRUNCATE test_zstd;
79 | SELECT count(DISTINCT test_zstd.*) FROM test_zstd;
80 | count
81 | -------
82 | 0
83 | (1 row)
84 |
85 | SET client_min_messages TO WARNING;
86 | DROP SCHEMA am_zstd CASCADE;
87 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/input/columnar_copyto.source:
--------------------------------------------------------------------------------
1 | --
2 | -- Test copying data from columnar tables.
3 | --
4 | CREATE TABLE test_contestant(handle TEXT, birthdate DATE, rating INT,
5 | percentile FLOAT, country CHAR(3), achievements TEXT[])
6 | USING columnar;
7 |
8 | -- load table data from file
9 | COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
10 |
11 | -- export using COPY table TO ...
12 | COPY test_contestant TO STDOUT;
13 |
14 | -- export using COPY (SELECT * FROM table) TO ...
15 | COPY (select * from test_contestant) TO STDOUT;
16 |
17 | DROP TABLE test_contestant CASCADE;
18 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/input/columnar_data_types.source:
--------------------------------------------------------------------------------
1 | --
2 | -- Test loading and reading different data types to/from columnar foreign tables.
3 | --
4 |
5 |
6 | -- Settings to make the result deterministic
7 | SET datestyle = "ISO, YMD";
8 | SET timezone to 'GMT';
9 | SET intervalstyle TO 'POSTGRES_VERBOSE';
10 |
11 |
12 | -- Test array types
13 | CREATE TABLE test_array_types (int_array int[], bigint_array bigint[],
14 | text_array text[]) USING columnar;
15 |
16 | COPY test_array_types FROM '@abs_srcdir@/data/array_types.csv' WITH CSV;
17 |
18 | SELECT * FROM test_array_types;
19 |
20 |
21 | -- Test date/time types
22 | CREATE TABLE test_datetime_types (timestamp timestamp,
23 | timestamp_with_timezone timestamp with time zone, date date, time time,
24 | interval interval) USING columnar;
25 |
26 | COPY test_datetime_types FROM '@abs_srcdir@/data/datetime_types.csv' WITH CSV;
27 |
28 | SELECT * FROM test_datetime_types;
29 |
30 |
31 | -- Test enum and composite types
32 | CREATE TYPE enum_type AS ENUM ('a', 'b', 'c');
33 | CREATE TYPE composite_type AS (a int, b text);
34 |
35 | CREATE TABLE test_enum_and_composite_types (enum enum_type,
36 | composite composite_type) USING columnar;
37 |
38 | COPY test_enum_and_composite_types FROM
39 | '@abs_srcdir@/data/enum_and_composite_types.csv' WITH CSV;
40 |
41 | SELECT * FROM test_enum_and_composite_types;
42 |
43 |
44 | -- Test range types
45 | CREATE TABLE test_range_types (int4range int4range, int8range int8range,
46 | numrange numrange, tsrange tsrange) USING columnar;
47 |
48 | COPY test_range_types FROM '@abs_srcdir@/data/range_types.csv' WITH CSV;
49 |
50 | SELECT * FROM test_range_types;
51 |
52 |
53 | -- Test other types
54 | CREATE TABLE test_other_types (bool boolean, bytea bytea, money money,
55 | inet inet, bitstring bit varying(5), uuid uuid, json json) USING columnar;
56 |
57 | COPY test_other_types FROM '@abs_srcdir@/data/other_types.csv' WITH CSV;
58 |
59 | SELECT * FROM test_other_types;
60 |
61 |
62 | -- Test null values
63 | CREATE TABLE test_null_values (a int, b int[], c composite_type)
64 | USING columnar;
65 |
66 | COPY test_null_values FROM '@abs_srcdir@/data/null_values.csv' WITH CSV;
67 |
68 | SELECT * FROM test_null_values;
69 |
70 | CREATE TABLE test_json(j json) USING columnar;
71 | INSERT INTO test_json SELECT ('{"att": ' || g::text || '}')::json from generate_series(1,1000000) g;
72 | SELECT * FROM test_json WHERE (j->'att')::text::int8 > 999990;
73 | DROP TABLE test_json;
74 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/input/columnar_load.source:
--------------------------------------------------------------------------------
1 | --
2 | -- Test loading data into columnar tables.
3 | --
4 |
5 | -- COPY with incorrect delimiter
6 | COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv'
7 | WITH DELIMITER '|'; -- ERROR
8 |
9 | -- COPY with invalid program
10 | COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR
11 |
12 | -- COPY into uncompressed table from file
13 | COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
14 |
15 | -- COPY into uncompressed table from program
16 | COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV;
17 |
18 | select
19 | version_major, version_minor, reserved_stripe_id, reserved_row_number
20 | from columnar_test_helpers.columnar_storage_info('contestant');
21 |
22 | -- COPY into compressed table
23 | COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
24 |
25 | -- COPY into compressed table from program
26 | COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv'
27 | WITH CSV;
28 |
29 | select
30 | version_major, version_minor, reserved_stripe_id, reserved_row_number
31 | from columnar_test_helpers.columnar_storage_info('contestant_compressed');
32 |
33 | -- Test column list
34 | CREATE TABLE famous_constants (id int, name text, value real)
35 | USING columnar;
36 | COPY famous_constants (value, name, id) FROM STDIN WITH CSV;
37 | 3.141,pi,1
38 | 2.718,e,2
39 | 0.577,gamma,3
40 | 5.291e-11,bohr radius,4
41 | \.
42 |
43 | COPY famous_constants (name, value) FROM STDIN WITH CSV;
44 | avagadro,6.022e23
45 | electron mass,9.109e-31
46 | proton mass,1.672e-27
47 | speed of light,2.997e8
48 | \.
49 |
50 | SELECT * FROM famous_constants ORDER BY id, name;
51 |
52 | SELECT * FROM columnar_test_helpers.chunk_group_consistency;
53 |
54 | DROP TABLE famous_constants;
55 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/output/columnar_copyto.source:
--------------------------------------------------------------------------------
1 | --
2 | -- Test copying data from columnar tables.
3 | --
4 | CREATE TABLE test_contestant(handle TEXT, birthdate DATE, rating INT,
5 | percentile FLOAT, country CHAR(3), achievements TEXT[])
6 | USING columnar;
7 | -- load table data from file
8 | COPY test_contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
9 | -- export using COPY table TO ...
10 | COPY test_contestant TO STDOUT;
11 | a 01-10-1990 2090 97.1 XA {a}
12 | b 11-01-1990 2203 98.1 XA {a,b}
13 | c 11-01-1988 2907 99.4 XB {w,y}
14 | d 05-05-1985 2314 98.3 XB {}
15 | e 05-05-1995 2236 98.2 XC {a}
16 | -- export using COPY (SELECT * FROM table) TO ...
17 | COPY (select * from test_contestant) TO STDOUT;
18 | a 01-10-1990 2090 97.1 XA {a}
19 | b 11-01-1990 2203 98.1 XA {a,b}
20 | c 11-01-1988 2907 99.4 XB {w,y}
21 | d 05-05-1985 2314 98.3 XB {}
22 | e 05-05-1995 2236 98.2 XC {a}
23 | DROP TABLE test_contestant CASCADE;
24 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/output/columnar_load.source:
--------------------------------------------------------------------------------
1 | --
2 | -- Test loading data into columnar tables.
3 | --
4 | -- COPY with incorrect delimiter
5 | COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv'
6 | WITH DELIMITER '|'; -- ERROR
7 | ERROR: missing data for column "birthdate"
8 | CONTEXT: COPY contestant, line 1: "a,1990-01-10,2090,97.1,XA ,{a}"
9 | -- COPY with invalid program
10 | COPY contestant FROM PROGRAM 'invalid_program' WITH CSV; -- ERROR
11 | ERROR: program "invalid_program" failed
12 | DETAIL: command not found
13 | -- COPY into uncompressed table from file
14 | COPY contestant FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
15 | -- COPY into uncompressed table from program
16 | COPY contestant FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv' WITH CSV;
17 | select
18 | version_major, version_minor, reserved_stripe_id, reserved_row_number
19 | from columnar_test_helpers.columnar_storage_info('contestant');
20 | version_major | version_minor | reserved_stripe_id | reserved_row_number
21 | ---------------+---------------+--------------------+---------------------
22 | 2 | 0 | 3 | 300001
23 | (1 row)
24 |
25 | -- COPY into compressed table
26 | COPY contestant_compressed FROM '@abs_srcdir@/data/contestants.1.csv' WITH CSV;
27 | -- COPY into compressed table from program
28 | COPY contestant_compressed FROM PROGRAM 'cat @abs_srcdir@/data/contestants.2.csv'
29 | WITH CSV;
30 | select
31 | version_major, version_minor, reserved_stripe_id, reserved_row_number
32 | from columnar_test_helpers.columnar_storage_info('contestant_compressed');
33 | version_major | version_minor | reserved_stripe_id | reserved_row_number
34 | ---------------+---------------+--------------------+---------------------
35 | 2 | 0 | 3 | 300001
36 | (1 row)
37 |
38 | -- Test column list
39 | CREATE TABLE famous_constants (id int, name text, value real)
40 | USING columnar;
41 | COPY famous_constants (value, name, id) FROM STDIN WITH CSV;
42 | COPY famous_constants (name, value) FROM STDIN WITH CSV;
43 | SELECT * FROM famous_constants ORDER BY id, name;
44 | id | name | value
45 | ----+----------------+-----------
46 | 1 | pi | 3.141
47 | 2 | e | 2.718
48 | 3 | gamma | 0.577
49 | 4 | bohr radius | 5.291e-11
50 | | avagadro | 6.022e+23
51 | | electron mass | 9.109e-31
52 | | proton mass | 1.672e-27
53 | | speed of light | 2.997e+08
54 | (8 rows)
55 |
56 | SELECT * FROM columnar_test_helpers.chunk_group_consistency;
57 | consistent
58 | ------------
59 | t
60 | (1 row)
61 |
62 | DROP TABLE famous_constants;
63 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/.gitignore:
--------------------------------------------------------------------------------
1 | /columnar_copyto.sql
2 | /columnar_data_types.sql
3 | /columnar_load.sql
4 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_alter_set_type.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- Testing ALTER TABLE on columnar tables.
3 | --
4 |
5 | CREATE TABLE test_alter_table (a int, b int, c text) USING columnar;
6 |
7 | WITH sample_data AS (VALUES
8 | (1, 2, '3'),
9 | (4, 5, '6')
10 | )
11 | INSERT INTO test_alter_table SELECT * FROM sample_data;
12 |
13 | WITH sample_data AS (VALUES
14 | (5, 9, '11'),
15 | (12, 83, '93')
16 | )
17 | INSERT INTO test_alter_table SELECT * FROM sample_data;
18 |
19 | ALTER TABLE test_alter_table ALTER COLUMN a TYPE jsonb USING row_to_json(row(a));
20 | SELECT * FROM test_alter_table ORDER BY a;
21 |
22 | ALTER TABLE test_alter_table ALTER COLUMN c TYPE int USING c::integer;
23 | SELECT sum(c) FROM test_alter_table;
24 |
25 | ALTER TABLE test_alter_table ALTER COLUMN b TYPE bigint;
26 | SELECT * FROM test_alter_table ORDER BY a;
27 |
28 | ALTER TABLE test_alter_table ALTER COLUMN b TYPE float USING (b::float + 0.5);
29 | SELECT * FROM test_alter_table ORDER BY a;
30 |
31 | DROP TABLE test_alter_table;
32 |
33 | -- Make sure that the correct table options are used when rewriting the table.
34 | -- This is reflected by the VACUUM VERBOSE output right after a rewrite showing
35 | -- that all chunks are compressed with the configured compression algorithm
36 | CREATE TABLE test(i int) USING columnar;
37 | SELECT columnar.alter_columnar_table_set('test', compression => 'lz4');
38 | INSERT INTO test VALUES(1);
39 | VACUUM VERBOSE test;
40 |
41 | ALTER TABLE test ALTER COLUMN i TYPE int8;
42 | VACUUM VERBOSE test;
43 |
44 | DROP TABLE test;
45 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_analyze.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- Test the ANALYZE command for columnar tables.
3 | --
4 |
5 | -- ANALYZE uncompressed table
6 | ANALYZE contestant;
7 | SELECT count(*) FROM pg_stats WHERE tablename='contestant';
8 |
9 | -- ANALYZE compressed table
10 | ANALYZE contestant_compressed;
11 | SELECT count(*) FROM pg_stats WHERE tablename='contestant_compressed';
12 |
13 | -- ANALYZE a table with lots of data to trigget qsort in analyze.c
14 | CREATE TABLE test_analyze(a int, b text, c char) USING columnar;
15 | INSERT INTO test_analyze SELECT floor(i / 1000), floor(i / 10)::text, 4 FROM generate_series(1, 100000) i;
16 | INSERT INTO test_analyze SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 110000) i;
17 |
18 | ANALYZE test_analyze;
19 | DROP TABLE test_analyze;
20 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_cache.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE big_table (
2 | id INT,
3 | firstname TEXT,
4 | lastname TEXT
5 | ) USING columnar;
6 |
7 | INSERT INTO big_table (id, firstname, lastname)
8 | SELECT i,
9 | CONCAT('firstname-', i),
10 | CONCAT('lastname-', i)
11 | FROM generate_series(1, 1000000) as i;
12 |
13 | -- get some baselines from multiple chunks
14 | SELECT firstname,
15 | lastname,
16 | SUM(id)
17 | FROM big_table
18 | WHERE id < 1000
19 | GROUP BY firstname,
20 | lastname
21 | UNION
22 | SELECT firstname,
23 | lastname,
24 | SUM(id)
25 | FROM big_table
26 | WHERE id BETWEEN 15000 AND 16000
27 | GROUP BY firstname,
28 | lastname
29 | ORDER BY firstname;
30 |
31 |
32 | -- enable caching
33 | SET columnar.enable_column_cache = 't';
34 |
35 | -- the results should be the same as above
36 | SELECT firstname,
37 | lastname,
38 | SUM(id)
39 | FROM big_table
40 | WHERE id < 1000
41 | GROUP BY firstname,
42 | lastname
43 | UNION
44 | SELECT firstname,
45 | lastname,
46 | SUM(id)
47 | FROM big_table
48 | WHERE id BETWEEN 15000 AND 16000
49 | GROUP BY firstname,
50 | lastname
51 | ORDER BY firstname;
52 |
53 | -- disable caching
54 | SET columnar.enable_column_cache = 'f';
55 |
56 | CREATE TABLE test_2 (
57 | value INT,
58 | updated_value INT
59 | ) USING columnar;
60 |
61 | INSERT INTO test_2 (value)
62 | SELECT generate_series(1, 1000000, 1);
63 |
64 | BEGIN;
65 | SELECT SUM(value)
66 | FROM test_2;
67 |
68 | UPDATE test_2
69 | SET updated_value = value * 2;
70 |
71 | SELECT SUM(updated_value)
72 | FROM test_2;
73 |
74 | DELETE FROM test_2
75 | WHERE value % 2 = 0;
76 |
77 | SELECT SUM(value)
78 | FROM test_2;
79 | COMMIT;
80 |
81 | DROP TABLE test_2;
82 |
83 | set columnar.enable_column_cache = 't';
84 |
85 | CREATE TABLE test_2 (
86 | value INT,
87 | updated_value INT
88 | ) USING columnar;
89 |
90 | INSERT INTO test_2 (value)
91 | SELECT generate_series(1, 1000000, 1);
92 |
93 | BEGIN;
94 | SELECT SUM(value)
95 | FROM test_2;
96 |
97 | UPDATE test_2
98 | SET updated_value = value * 2;
99 |
100 | SELECT SUM(updated_value)
101 | FROM test_2;
102 |
103 | DELETE FROM test_2
104 | WHERE value % 2 = 0;
105 |
106 | SELECT SUM(value)
107 | FROM test_2;
108 | COMMIT;
109 |
110 | DROP TABLE test_2;
111 |
112 | CREATE TABLE t1 (i int) USING columnar;
113 |
114 | INSERT INTO t1 SELECT generate_series(1, 1000000, 1);
115 | EXPLAIN SELECT COUNT(*) FROM t1;
116 | DROP TABLE t1;
117 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_clean.sql:
--------------------------------------------------------------------------------
1 |
2 | DROP TABLE test_null_values;
3 | DROP TABLE test_other_types;
4 | DROP TABLE test_range_types;
5 | DROP TABLE test_enum_and_composite_types;
6 | DROP TYPE composite_type;
7 | DROP TYPE enum_type;
8 | DROP TABLE test_datetime_types;
9 | DROP TABLE test_array_types;
10 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_cursor.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- Testing cursors on columnar tables.
3 | --
4 |
5 | CREATE TABLE test_cursor (a int, b int) USING columnar;
6 | INSERT INTO test_cursor SELECT i, j FROM generate_series(0, 100)i, generate_series(100, 200)j;
7 |
8 | -- A case where the WHERE clause might filter out some chunks
9 | EXPLAIN (analyze on, costs off, timing off, summary off) SELECT * FROM test_cursor WHERE a = 25;
10 | BEGIN;
11 | DECLARE a_25 SCROLL CURSOR
12 | FOR SELECT * FROM test_cursor WHERE a = 25 ORDER BY 2;
13 | FETCH 3 FROM a_25;
14 | FETCH PRIOR FROM a_25;
15 | FETCH NEXT FROM a_25;
16 | FETCH NEXT FROM a_25;
17 | FETCH RELATIVE -2 FROM a_25;
18 | FETCH LAST FROM a_25;
19 | FETCH RELATIVE -25 FROM a_25;
20 | MOVE a_25;
21 | FETCH a_25;
22 | MOVE LAST FROM a_25;
23 | FETCH a_25;
24 | MOVE RELATIVE -3 FROM a_25;
25 | FETCH a_25;
26 | MOVE RELATIVE -3 FROM a_25;
27 | FETCH a_25;
28 | MOVE FORWARD 2 FROM a_25;
29 | FETCH a_25;
30 | MOVE RELATIVE -3 FROM a_25;
31 | FETCH a_25;
32 | UPDATE test_cursor SET a = 8000 WHERE CURRENT OF a_25;
33 | COMMIT;
34 |
35 | -- A case where the WHERE clause doesn't filter out any chunks
36 | EXPLAIN (analyze on, costs off, timing off, summary off) SELECT * FROM test_cursor WHERE a > 25;
37 | BEGIN;
38 | DECLARE a_25 SCROLL CURSOR
39 | FOR SELECT * FROM test_cursor WHERE a > 25 ORDER BY 1, 2;
40 | FETCH 3 FROM a_25;
41 | FETCH PRIOR FROM a_25;
42 | FETCH NEXT FROM a_25;
43 | FETCH NEXT FROM a_25;
44 | FETCH RELATIVE -2 FROM a_25;
45 | FETCH LAST FROM a_25;
46 | FETCH RELATIVE -25 FROM a_25;
47 | MOVE a_25;
48 | FETCH a_25;
49 | MOVE LAST FROM a_25;
50 | FETCH a_25;
51 | MOVE RELATIVE -3 FROM a_25;
52 | FETCH a_25;
53 | MOVE RELATIVE -3 FROM a_25;
54 | FETCH a_25;
55 | MOVE FORWARD 2 FROM a_25;
56 | FETCH a_25;
57 | MOVE RELATIVE -3 FROM a_25;
58 | FETCH a_25;
59 | UPDATE test_cursor SET a = 8000 WHERE CURRENT OF a_25;
60 | COMMIT;
61 |
62 | DROP TABLE test_cursor CASCADE;
63 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_customindex.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- Test custom index: Test shows only that we are exchanging Index scan with CustomIndexScan while
3 | -- all other information is still the same. The feature is not visible through EXPLAIN because output
4 | -- will be same in both cases, but difference is noticeable how the columnar storage engine is read.
5 | -- With index scan ALL columns are requested from storage (this can be significant overhead in performance)
6 | -- while with CustomIndexScan we will request only columns that are needed - that are going to be used as output.
7 | -- Test also shows that we are not using CustomIndexScan with heap tables and only on columnar tables.
8 | --
9 |
10 | SET columnar.enable_columnar_index_scan TO TRUE;
11 |
12 | CREATE TABLE t(a INT PRIMARY KEY, b INT, c TEXT) USING columnar;
13 | CREATE INDEX t_idx ON t USING btree(b);
14 |
15 | CREATE TABLE t_heap(a INT PRIMARY KEY, b INT, c TEXT);
16 | CREATE INDEX t_idx_heap ON t_heap USING btree(b);
17 |
18 | INSERT INTO t SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g;
19 | INSERT INTO t_heap SELECT g, g % 20, 'abcde_' || g FROM generate_series(1, 300000) g;
20 |
21 | -- make sure that we test index scan
22 | set columnar.enable_custom_scan TO 'off';
23 | set enable_seqscan TO off;
24 | set seq_page_cost TO 10000000;
25 |
26 | EXPLAIN (VERBOSE) SELECT a FROM t WHERE b > 18 ORDER BY b LIMIT 10;
27 |
28 | EXPLAIN (VERBOSE) SELECT a FROM t_heap WHERE b > 18 ORDER BY b LIMIT 10;
29 |
30 | DROP TABLE t;
31 | DROP TABLE t_heap;
32 |
33 | SET columnar.enable_custom_index_scan TO default;
34 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_drop.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- Tests the different DROP commands for columnar tables.
3 | --
4 | -- DROP TABL
5 | -- DROP SCHEMA
6 | -- DROP EXTENSION
7 | -- DROP DATABASE
8 | --
9 |
10 | -- Note that travis does not create
11 | -- citus extension in default database (postgres). This has caused
12 | -- different behavior between travis tests and local tests. Thus
13 | -- 'postgres' directory is excluded from comparison to have the same result.
14 |
15 | -- store postgres database oid
16 | SELECT oid postgres_oid FROM pg_database WHERE datname = 'postgres' \gset
17 |
18 | SELECT count(distinct storage_id) AS columnar_stripes_before_drop FROM columnar.stripe \gset
19 |
20 | -- DROP columnar tables
21 | DROP TABLE contestant;
22 | DROP TABLE contestant_compressed;
23 |
24 | -- make sure DROP deletes metadata
25 | SELECT :columnar_stripes_before_drop - count(distinct storage_id) FROM columnar.stripe;
26 |
27 | -- Create a columnar table under a schema and drop it.
28 | CREATE SCHEMA test_schema;
29 | CREATE TABLE test_schema.test_table(data int) USING columnar;
30 | INSERT INTO test_schema.test_table VALUES (1);
31 |
32 | SELECT count(*) AS columnar_stripes_before_drop FROM columnar.stripe \gset
33 | DROP SCHEMA test_schema CASCADE;
34 | SELECT :columnar_stripes_before_drop - count(distinct storage_id) FROM columnar.stripe;
35 |
36 | SELECT current_database() datname \gset
37 |
38 | CREATE DATABASE db_to_drop;
39 | \c db_to_drop
40 | CREATE EXTENSION columnar;
41 | SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
42 |
43 | CREATE TABLE test_table(data int) USING columnar;
44 |
45 | DROP EXTENSION columnar CASCADE;
46 |
47 | -- test database drop
48 | CREATE EXTENSION columnar;
49 | SELECT oid::text databaseoid FROM pg_database WHERE datname = current_database() \gset
50 |
51 | CREATE TABLE test_table(data int) USING columnar;
52 |
53 | \c :datname
54 |
55 | DROP DATABASE db_to_drop;
56 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_empty.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- Test different operations on empty columnar tables.
3 | --
4 |
5 | SET columnar.compression to 'none';
6 | create table t_uncompressed(a int) using columnar;
7 | create table t_compressed(a int) using columnar;
8 |
9 | -- set options
10 | SELECT columnar.alter_columnar_table_set('t_compressed', compression => 'pglz');
11 | SELECT columnar.alter_columnar_table_set('t_compressed', stripe_row_limit => 2000);
12 | SELECT columnar.alter_columnar_table_set('t_compressed', chunk_group_row_limit => 1000);
13 |
14 | SELECT * FROM columnar.options WHERE regclass = 't_compressed'::regclass;
15 |
16 | -- select
17 | select * from t_uncompressed;
18 | select count(*) from t_uncompressed;
19 | select * from t_compressed;
20 | select count(*) from t_compressed;
21 |
22 | -- check storage
23 | select
24 | version_major, version_minor, reserved_stripe_id, reserved_row_number
25 | from columnar_test_helpers.columnar_storage_info('t_compressed');
26 | select
27 | version_major, version_minor, reserved_stripe_id, reserved_row_number
28 | from columnar_test_helpers.columnar_storage_info('t_uncompressed');
29 |
30 | -- explain
31 | explain (costs off, summary off, timing off) select * from t_uncompressed;
32 | explain (costs off, summary off, timing off) select * from t_compressed;
33 |
34 | -- vacuum
35 | vacuum t_compressed;
36 | vacuum t_uncompressed;
37 |
38 | -- vacuum full
39 | vacuum full t_compressed;
40 | vacuum full t_uncompressed;
41 |
42 | -- check storage
43 | select
44 | version_major, version_minor, reserved_stripe_id, reserved_row_number
45 | from columnar_test_helpers.columnar_storage_info('t_compressed');
46 | select
47 | version_major, version_minor, reserved_stripe_id, reserved_row_number
48 | from columnar_test_helpers.columnar_storage_info('t_uncompressed');
49 |
50 | -- analyze
51 | analyze t_uncompressed;
52 | analyze t_compressed;
53 |
54 | -- truncate
55 | truncate t_uncompressed;
56 | truncate t_compressed;
57 |
58 | -- alter type
59 | alter table t_uncompressed alter column a type text;
60 | alter table t_compressed alter column a type text;
61 |
62 | -- check storage
63 | select
64 | version_major, version_minor, reserved_stripe_id, reserved_row_number
65 | from columnar_test_helpers.columnar_storage_info('t_compressed');
66 | select
67 | version_major, version_minor, reserved_stripe_id, reserved_row_number
68 | from columnar_test_helpers.columnar_storage_info('t_uncompressed');
69 |
70 | -- verify cost of scanning an empty table is zero, not NaN
71 | explain table t_uncompressed;
72 | explain table t_compressed;
73 |
74 | -- drop
75 | drop table t_compressed;
76 | drop table t_uncompressed;
77 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_fallback_scan.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- columnar_fallback_scan.sql
3 | --
4 | -- Test columnar.enable_custom_scan = false, which will use an
5 | -- ordinary sequential scan. It won't benefit from projection pushdown
6 | -- or qual pushdown, but it should return the correct results.
7 | --
8 |
9 | set columnar.enable_custom_scan = false;
10 |
11 | create table fallback_scan(i int) using columnar;
12 | -- large enough to test parallel_workers > 1
13 | select columnar.alter_columnar_table_set('fallback_scan', compression => 'none');
14 | insert into fallback_scan select generate_series(1,150000);
15 | vacuum analyze fallback_scan;
16 |
17 | select count(*), min(i), max(i), avg(i) from fallback_scan;
18 |
19 | --
20 | -- Negative test: try to force a parallel plan with at least two
21 | -- workers, but columnar should reject it and use a non-parallel scan.
22 | --
23 | set min_parallel_table_scan_size = 1;
24 | set parallel_tuple_cost = 0;
25 | set max_parallel_workers = 4;
26 | set max_parallel_workers_per_gather = 4;
27 | explain (costs off) select count(*), min(i), max(i), avg(i) from fallback_scan;
28 | select count(*), min(i), max(i), avg(i) from fallback_scan;
29 |
30 | set min_parallel_table_scan_size to default;
31 | set parallel_tuple_cost to default;
32 | set max_parallel_workers to default;
33 | set max_parallel_workers_per_gather to default;
34 |
35 | set columnar.enable_custom_scan to default;
36 |
37 | drop table fallback_scan;
38 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_first_row_number.sql:
--------------------------------------------------------------------------------
1 | CREATE SCHEMA columnar_first_row_number;
2 | SET search_path tO columnar_first_row_number;
3 |
4 | CREATE TABLE col_table_1 (a int) USING columnar;
5 |
6 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 10) i;
7 |
8 | BEGIN;
9 | -- we don't use same first_row_number even if the xact is rollback'ed
10 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 11) i;
11 | ROLLBACK;
12 |
13 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 12) i;
14 |
15 | SELECT columnar.alter_columnar_table_set('col_table_1', stripe_row_limit => 1000);
16 |
17 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 2350) i;
18 |
19 | SELECT row_count, first_row_number FROM columnar.stripe a
20 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass)
21 | ORDER BY stripe_num;
22 |
23 | VACUUM FULL col_table_1;
24 |
25 | -- show that we properly update first_row_number after VACUUM FULL
26 | SELECT row_count, first_row_number FROM columnar.stripe a
27 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass)
28 | ORDER BY stripe_num;
29 |
30 | TRUNCATE col_table_1;
31 |
32 | BEGIN;
33 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 16) i;
34 | INSERT INTO col_table_1 SELECT i FROM generate_series(1, 16) i;
35 | COMMIT;
36 |
37 | -- show that we start with first_row_number=1 after TRUNCATE
38 | SELECT row_count, first_row_number FROM columnar.stripe a
39 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid('col_table_1'::regclass)
40 | ORDER BY stripe_num;
41 |
42 | SET client_min_messages TO ERROR;
43 | DROP SCHEMA columnar_first_row_number CASCADE;
44 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_join.sql:
--------------------------------------------------------------------------------
1 | CREATE SCHEMA am_columnar_join;
2 | SET search_path TO am_columnar_join;
3 |
4 | CREATE TABLE users (id int, name text) USING columnar;
5 | INSERT INTO users SELECT a, 'name' || a FROM generate_series(0,30-1) AS a;
6 |
7 | CREATE TABLE things (id int, user_id int, name text) USING columnar;
8 | INSERT INTO things SELECT a, a % 30, 'thing' || a FROM generate_series(1,300) AS a;
9 |
10 | -- force the nested loop to rescan the table
11 | SET enable_material TO off;
12 | SET enable_hashjoin TO off;
13 | SET enable_mergejoin TO off;
14 |
15 | SELECT count(*)
16 | FROM users
17 | JOIN things ON (users.id = things.user_id)
18 | WHERE things.id > 290;
19 |
20 | -- verify the join uses a nested loop to trigger the rescan behaviour
21 | EXPLAIN (COSTS OFF)
22 | SELECT count(*)
23 | FROM users
24 | JOIN things ON (users.id = things.user_id)
25 | WHERE things.id > 299990;
26 |
27 | EXPLAIN (COSTS OFF)
28 | SELECT u1.id, u2.id, COUNT(u2.*)
29 | FROM users u1
30 | JOIN users u2 ON (u1.id::text = u2.name)
31 | WHERE u2.id > 299990
32 | GROUP BY u1.id, u2.id;
33 |
34 | SET client_min_messages TO warning;
35 | DROP SCHEMA am_columnar_join CASCADE;
36 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_lz4.sql:
--------------------------------------------------------------------------------
1 | SELECT columnar_test_helpers.compression_type_supported('lz4') AS lz4_supported \gset
2 | \if :lz4_supported
3 | \else
4 | \q
5 | \endif
6 |
7 | CREATE SCHEMA am_alz4;
8 | SET search_path TO am_alz4;
9 |
10 | SET columnar.compression TO 'lz4';
11 | CREATE TABLE test_lz4 (a int, b text, c int) USING columnar;
12 |
13 | INSERT INTO test_lz4 SELECT floor(i / 1000), floor(i / 10)::text, 4 FROM generate_series(1, 10000) i;
14 | SELECT count(*) FROM test_lz4;
15 |
16 | INSERT INTO test_lz4 SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 11000) i;
17 | SELECT count(*) FROM test_lz4;
18 |
19 | SELECT pg_relation_size('test_lz4') AS size_lz4 \gset
20 |
21 | SELECT DISTINCT * FROM test_lz4 ORDER BY a, b, c LIMIT 5;
22 |
23 | -- compare compression rate to pglz
24 | SET columnar.compression TO 'pglz';
25 | CREATE TABLE test_pglz (LIKE test_lz4) USING columnar;
26 | INSERT INTO test_pglz SELECT * FROM test_lz4;
27 |
28 | SELECT pg_relation_size('test_pglz') AS size_pglz \gset
29 |
30 | -- verify that pglz & lz4 resulted in different compression ratios
31 | SELECT :size_pglz <> :size_lz4;
32 |
33 | -- Other operations
34 | VACUUM FULL test_lz4;
35 | ANALYZE test_lz4;
36 |
37 | SELECT count(DISTINCT test_lz4.*) FROM test_lz4;
38 |
39 | TRUNCATE test_lz4;
40 |
41 | SELECT count(DISTINCT test_lz4.*) FROM test_lz4;
42 |
43 | SET client_min_messages TO WARNING;
44 | DROP SCHEMA am_alz4 CASCADE;
45 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_matview.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- Testing we materialized views properly
3 | --
4 |
5 | SET columnar.compression TO 'none';
6 |
7 | CREATE TABLE t(a int, b int) USING columnar;
8 |
9 | INSERT INTO t SELECT floor(i / 4), 2 * i FROM generate_series(1, 10) i;
10 |
11 | CREATE MATERIALIZED VIEW t_view(a, bsum, cnt) USING columnar AS
12 | SELECT a, sum(b), count(*) FROM t GROUP BY a;
13 |
14 | SELECT * FROM t_view a ORDER BY a;
15 |
16 | INSERT INTO t SELECT floor(i / 4), 2 * i FROM generate_series(11, 20) i;
17 |
18 | SELECT * FROM t_view a ORDER BY a;
19 |
20 | -- show columnar options for materialized view
21 | SELECT * FROM columnar.options
22 | WHERE regclass = 't_view'::regclass;
23 |
24 | -- show we can set options on a materialized view
25 | SELECT columnar.alter_columnar_table_set('t_view', compression => 'pglz');
26 | SELECT * FROM columnar.options
27 | WHERE regclass = 't_view'::regclass;
28 |
29 | REFRESH MATERIALIZED VIEW t_view;
30 |
31 | -- verify options have not been changed
32 | SELECT * FROM columnar.options
33 | WHERE regclass = 't_view'::regclass;
34 |
35 | SELECT * FROM t_view a ORDER BY a;
36 |
37 | -- verify that we have created metadata entries for the materialized view
38 | SELECT columnar_test_helpers.columnar_relation_storageid(oid) AS storageid
39 | FROM pg_class WHERE relname='t_view' \gset
40 |
41 | SELECT count(*) FROM columnar.stripe WHERE storage_id=:storageid;
42 | SELECT count(*) FROM columnar.chunk WHERE storage_id=:storageid;
43 |
44 | DROP TABLE t CASCADE;
45 |
46 | -- dropping must remove metadata
47 | SELECT count(*) FROM columnar.stripe WHERE storage_id=:storageid;
48 | SELECT count(*) FROM columnar.chunk WHERE storage_id=:storageid;
49 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_permissions.sql:
--------------------------------------------------------------------------------
1 |
2 | select current_user \gset
3 |
4 | create user columnar_user;
5 | grant all on schema public to columnar_user;
6 |
7 | \c - columnar_user
8 |
9 | create table columnar_permissions(i int) using columnar;
10 | insert into columnar_permissions values(1);
11 | alter table columnar_permissions add column j int;
12 | insert into columnar_permissions values(2,20);
13 | vacuum columnar_permissions;
14 | truncate columnar_permissions;
15 | drop table columnar_permissions;
16 |
17 | \c - :current_user
18 |
19 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_rollback.sql:
--------------------------------------------------------------------------------
1 | --
2 | -- Testing we handle rollbacks properly
3 | --
4 |
5 | CREATE TABLE t(a int, b int) USING columnar;
6 |
7 | CREATE VIEW t_stripes AS
8 | SELECT * FROM columnar.stripe a, pg_class b
9 | WHERE a.storage_id = columnar_test_helpers.columnar_relation_storageid(b.oid) AND b.relname = 't';
10 |
11 | BEGIN;
12 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i;
13 | ROLLBACK;
14 | SELECT count(*) FROM t;
15 |
16 | select
17 | version_major, version_minor, reserved_stripe_id, reserved_row_number
18 | from columnar_test_helpers.columnar_storage_info('t');
19 |
20 | -- check stripe metadata also have been rolled-back
21 | SELECT count(*) FROM t_stripes;
22 |
23 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i;
24 | SELECT count(*) FROM t;
25 |
26 | SELECT count(*) FROM t_stripes;
27 |
28 | -- savepoint rollback
29 | BEGIN;
30 | SAVEPOINT s0;
31 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i;
32 | SELECT count(*) FROM t; -- force flush
33 | SAVEPOINT s1;
34 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i;
35 |
36 | select
37 | version_major, version_minor, reserved_stripe_id, reserved_row_number
38 | from columnar_test_helpers.columnar_storage_info('t');
39 |
40 | SELECT count(*) FROM t;
41 | ROLLBACK TO SAVEPOINT s1;
42 | SELECT count(*) FROM t;
43 | ROLLBACK TO SAVEPOINT s0;
44 | SELECT count(*) FROM t;
45 | INSERT INTO t SELECT i, i+1 FROM generate_series(1, 10) i;
46 | COMMIT;
47 |
48 | select
49 | version_major, version_minor, reserved_stripe_id, reserved_row_number
50 | from columnar_test_helpers.columnar_storage_info('t');
51 |
52 | SELECT count(*) FROM t;
53 |
54 | SELECT count(*) FROM t_stripes;
55 |
56 | DROP TABLE t;
57 | DROP VIEW t_stripes;
58 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_upsert.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE upsert_test (
2 | i INT,
3 | t TEXT
4 | ) USING columnar;
5 |
6 | CREATE UNIQUE INDEX ON upsert_test (t);
7 |
8 | INSERT INTO upsert_test (i, t) VALUES (1, 'hello');
9 | INSERT INTO upsert_test (i, t) VALUES (2, 'world');
10 |
11 | -- should work, then roll back
12 | BEGIN;
13 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'foo';
14 | SELECT * FROM upsert_test;
15 | ROLLBACK;
16 |
17 | SELECT * FROM upsert_test;
18 |
19 | -- should take affect immediately
20 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'bar';
21 | SELECT * FROM upsert_test;
22 |
23 | -- should work as expected
24 | BEGIN;
25 | INSERT INTO upsert_test (t) VALUES ( 'world' ) ON CONFLICT (t) DO UPDATE SET t = 'foo';
26 | SELECT * FROM upsert_test;
27 | COMMIT;
28 |
29 | -- should also work as expected, a select can mask a bug
30 | DELETE FROM upsert_test;
31 | BEGIN;
32 | INSERT INTO upsert_test (i, t) VALUES (1, 'hello');
33 | INSERT INTO upsert_test (t) VALUES ( 'hello' ) ON CONFLICT (t) DO UPDATE SET t = 'bar';
34 | COMMIT;
35 |
36 | SELECT * FROM upsert_test;
37 |
38 | DROP TABLE upsert_test;
39 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_vectorization.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE t (id int, ts timestamp) USING columnar;
2 | INSERT INTO t SELECT id, now() + id * '1 day'::interval FROM generate_series(1, 100000) id;
3 | EXPLAIN (costs off) SELECT * FROM t WHERE ts between '2026-01-01'::timestamp and '2026-02-01'::timestamp;
4 | DROP TABLE t;
5 |
6 | CREATE TABLE t (id int, ts timestamptz) USING columnar;
7 | INSERT INTO t SELECT id, now() + id * '1 day'::interval FROM generate_series(1, 100000) id;
8 | EXPLAIN (costs off) SELECT * FROM t WHERE ts between '2026-01-01'::timestamptz and '2026-02-01'::timestamptz;
9 | DROP TABLE t;
10 |
--------------------------------------------------------------------------------
/columnar/src/test/regress/sql/columnar_zstd.sql:
--------------------------------------------------------------------------------
1 | SELECT columnar_test_helpers.compression_type_supported('zstd') AS zstd_supported \gset
2 | \if :zstd_supported
3 | \else
4 | \q
5 | \endif
6 |
7 | CREATE SCHEMA am_zstd;
8 | SET search_path TO am_zstd;
9 |
10 | SET columnar.compression TO 'zstd';
11 | CREATE TABLE test_zstd (a int, b text, c int) USING columnar;
12 |
13 | INSERT INTO test_zstd SELECT i % 1000, (i % 10)::text, 4 FROM generate_series(1, 10000) i;
14 | SELECT count(*) FROM test_zstd;
15 |
16 | INSERT INTO test_zstd SELECT floor(i / 2), floor(i / 10)::text, 5 FROM generate_series(1000, 11000) i;
17 | SELECT count(*) FROM test_zstd;
18 |
19 | CREATE TABLE test_none (LIKE test_zstd) USING columnar;
20 | INSERT INTO test_none SELECT * FROM test_zstd;
21 |
22 | SELECT DISTINCT * FROM test_zstd ORDER BY a, b, c LIMIT 5;
23 |
24 | VACUUM FULL test_zstd;
25 |
26 | -- Hydra: we need to get `data_length` from stripe metadata information to compare compression
27 | -- rather than calling pg_relation_size function
28 |
29 | SELECT data_length AS size_comp_level_default FROM columnar.stripe WHERE storage_id = (
30 | SELECT storage_id from columnar_test_helpers.columnar_storage_info('test_zstd')) \gset
31 |
32 | -- change compression level
33 | SELECT columnar.alter_columnar_table_set('test_zstd', compression_level => 19);
34 | VACUUM FULL test_zstd;
35 |
36 | SELECT data_length AS size_comp_level_19 FROM columnar.stripe WHERE storage_id = (
37 | SELECT storage_id from columnar_test_helpers.columnar_storage_info('test_zstd')) \gset
38 |
39 | -- verify that higher compression level compressed better
40 | SELECT :size_comp_level_default > :size_comp_level_19 AS size_changed;
41 |
42 | -- compare compression rate to pglz
43 | SET columnar.compression TO 'pglz';
44 | CREATE TABLE test_pglz (LIKE test_zstd) USING columnar;
45 | INSERT INTO test_pglz SELECT * FROM test_zstd;
46 |
47 | SELECT pg_relation_size('test_pglz') AS size_pglz \gset
48 |
49 | -- verify that zstd compressed better than pglz
50 | SELECT :size_pglz > :size_comp_level_default;
51 |
52 | -- Other operations
53 |
54 | ANALYZE test_zstd;
55 | SELECT count(DISTINCT test_zstd.*) FROM test_zstd;
56 |
57 | TRUNCATE test_zstd;
58 |
59 | SELECT count(DISTINCT test_zstd.*) FROM test_zstd;
60 |
61 | SET client_min_messages TO WARNING;
62 | DROP SCHEMA am_zstd CASCADE;
63 |
--------------------------------------------------------------------------------
/columnar/vendor/README.md:
--------------------------------------------------------------------------------
1 | # Updating vendored dependencies
2 |
3 | ```bash
4 | rm -rf safestringlib
5 | git clone https://github.com/intel/safestringlib
6 | rm -rf safestringlib/{.git,unittests}
7 | git add safestringlib/
8 | git commit -m "Update safestringlib"
9 | git cherry-pick -x dc2a371d9f8b28ad0e68c5230bb998b4463d8131
10 | ```
11 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 |
4 | # Custom for Visual Studio
5 | *.cs diff=csharp
6 |
7 | # Standard to msysgit
8 | *.doc diff=astextplain
9 | *.DOC diff=astextplain
10 | *.docx diff=astextplain
11 | *.DOCX diff=astextplain
12 | *.dot diff=astextplain
13 | *.DOT diff=astextplain
14 | *.pdf diff=astextplain
15 | *.PDF diff=astextplain
16 | *.rtf diff=astextplain
17 | *.RTF diff=astextplain
18 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2014-2018 Intel Corporation
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
23 | ================================================================================
24 |
25 | Copyright (C) 2012, 2013 Cisco Systems
26 | All rights reserved.
27 |
28 | Permission is hereby granted, free of charge, to any person
29 | obtaining a copy of this software and associated documentation
30 | files (the "Software"), to deal in the Software without
31 | restriction, including without limitation the rights to use,
32 | copy, modify, merge, publish, distribute, sublicense, and/or
33 | sell copies of the Software, and to permit persons to whom the
34 | Software is furnished to do so, subject to the following
35 | conditions:
36 |
37 | The above copyright notice and this permission notice shall be
38 | included in all copies or substantial portions of the Software.
39 |
40 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
41 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
42 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
43 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
44 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
45 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
46 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
47 | OTHER DEALINGS IN THE SOFTWARE.
48 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/LICENSE©ING.txt:
--------------------------------------------------------------------------------
1 | Safe C Library
2 |
3 | Copyright (c) 2014-2016, Intel Corporation. All rights reserved.
4 |
5 | * Permission is hereby granted, free of charge, to any person
6 | * obtaining a copy of this software and associated documentation
7 | * files (the "Software"), to deal in the Software without
8 | * restriction, including without limitation the rights to use,
9 | * copy, modify, merge, publish, distribute, sublicense, and/or
10 | * sell copies of the Software, and to permit persons to whom the
11 | * Software is furnished to do so, subject to the following
12 | * conditions:
13 | *
14 | * The above copyright notice and this permission notice shall be
15 | * included in all copies or substantial portions of the Software.
16 | *
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
19 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
21 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
22 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 | * OTHER DEALINGS IN THE SOFTWARE.
25 |
26 | ================================================================================
27 |
28 | Copyright (C) 2012, 2013 Cisco Systems
29 | All rights reserved.
30 |
31 | Permission is hereby granted, free of charge, to any person
32 | obtaining a copy of this software and associated documentation
33 | files (the "Software"), to deal in the Software without
34 | restriction, including without limitation the rights to use,
35 | copy, modify, merge, publish, distribute, sublicense, and/or
36 | sell copies of the Software, and to permit persons to whom the
37 | Software is furnished to do so, subject to the following
38 | conditions:
39 |
40 | The above copyright notice and this permission notice shall be
41 | included in all copies or substantial portions of the Software.
42 |
43 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
44 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
45 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
46 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
47 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
48 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
49 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
50 | OTHER DEALINGS IN THE SOFTWARE.
51 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/README.md:
--------------------------------------------------------------------------------
1 | # safestringlib
2 | The Secure Development Lifecycle (SDL) recommends banning certain C Library functions because they directly contribute
3 | to security vulnerabilities such as buffer overflows. However routines for the manipulation of strings and memory buffers
4 | are common in software and firmware, and are essential to accomplish certain programming tasks. Safer replacements for
5 | these functions that avoid or prevent serious security vulnerabilities (e.g. buffer overflows, string format attacks,
6 | conversion overflows/underflows, etc.) are available in the SafeString Library.
7 |
8 | This library includes routines for safe string operations (like strcpy) and memory routines (like memcpy) that are
9 | recommended for Linux/Android operating systems, and will also work for Windows. This library is especially useful for
10 | cross-platform situations where one library for these routines is preferred.
11 |
12 | The Safe String Library is based on the Safe C Library by Cisco, and includes replacement C Library functions for the SDL
13 | banned functions, as well as a number of additional useful routines that are also susceptible to buffer overflows. This
14 | library continues to be made available under the MIT Open Source License.
15 |
16 | Cisco's Safe C Library was extended by Intel's Security Center of Excellence (SeCoE) to add additional routines, and
17 | include additional unit tests.
18 |
19 | LIST OF PRIMARY FUNCTIONS:
20 | -----------------------------
21 | * memcmp_s()
22 | * memcpy_s()
23 | * memmove_s()
24 | * memset_s()
25 | * memzero_s()
26 |
27 | * stpcpy_s()
28 | * stpncpy_s()
29 | * strcat_s()
30 | * strcpy_s()
31 | * strcspn_s()
32 | * strncat_s()
33 | * strncpy_s()
34 | * strnlen_s()
35 | * strpbrk_s()
36 | * strspn_s()
37 | * strstr_s()
38 | * strtok_s()
39 |
40 | * wcpcpy_s()
41 | * wcscat_s()
42 | * wcscpy_s()
43 | * wcsncat_s()
44 | * wcsnlen_s()
45 | * wmemcmp_s()
46 | * wmemcpy_s()
47 | * wmemmove_s()
48 | * wmemset_s()
49 |
50 |
51 | LIST OF ADDITIONAL STRING ROUTINES:
52 | ------------------------------------
53 | * strcasecmp_s()
54 | * strcasestr_s()
55 | * strcmp_s()
56 | * strcmpfld_s()
57 | * strcpyfld_s()
58 | * strcpyfldin_s()
59 | * strcpyfldout_s()
60 | * strfirstchar_s()
61 | * strfirstdiff_s()
62 | * strfirstsmae_s()
63 | * strisalphanumeric_s()
64 | * strisascii_s()
65 | * strisdigit_s()
66 | * strishes_s()
67 | * strislowercase_s()
68 | * strismixedcase_s()
69 | * strispassword_s()
70 | * strisuppercase_s()
71 | * strlastchar_s()
72 | * strlastdiff_s()
73 | * strlastsame_s()
74 | * strljustify_s()
75 | * strnterminate_s()
76 | * strprefix_s()
77 | * stremovews_s()
78 | * strtolowercase_s()
79 | * strtouppercase_s()
80 | * strzero_s()
81 |
82 |
83 | PLANNED ENHANCEMENTS:
84 | ----------------------
85 | - Add full sprintf_s() support
86 | - Add full sscanf_s() support
87 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/include/safe_lib.h:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * safe_lib.h -- Safe C Library
3 | *
4 | * October 2008, Bo Berry
5 | * Modified 2012, Jonathan Toppins <jtoppins@users.sourceforge.net>
6 | *
7 | * Copyright (c) 2008-2013 by Cisco Systems, Inc
8 | * All rights reserved.
9 | *
10 | * Permission is hereby granted, free of charge, to any person
11 | * obtaining a copy of this software and associated documentation
12 | * files (the "Software"), to deal in the Software without
13 | * restriction, including without limitation the rights to use,
14 | * copy, modify, merge, publish, distribute, sublicense, and/or
15 | * sell copies of the Software, and to permit persons to whom the
16 | * Software is furnished to do so, subject to the following
17 | * conditions:
18 | *
19 | * The above copyright notice and this permission notice shall be
20 | * included in all copies or substantial portions of the Software.
21 | *
22 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
24 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
26 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
27 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
29 | * OTHER DEALINGS IN THE SOFTWARE.
30 | *------------------------------------------------------------------
31 | */
32 |
33 | #ifndef __SAFE_LIB_H__
34 | #define __SAFE_LIB_H__
35 |
36 | #ifdef __cplusplus
37 | extern "C" {
38 | #endif
39 |
40 | #include "safe_types.h"
41 | #include "safe_lib_errno.h"
42 |
43 | /* C11 appendix K types - specific for bounds checking */
44 | typedef size_t rsize_t;
45 |
46 | /*
47 | * We depart from the standard and allow memory and string operations to
48 | * have different max sizes. See the respective safe_mem_lib.h or
49 | * safe_str_lib.h files.
50 | */
51 | /* #define RSIZE_MAX (~(rsize_t)0) - leave here for completeness */
52 |
53 | typedef void (*constraint_handler_t) (const char * /* msg */,
54 | void * /* ptr */,
55 | errno_t /* error */);
56 |
57 | extern void abort_handler_s(const char *msg, void *ptr, errno_t error);
58 | extern void ignore_handler_s(const char *msg, void *ptr, errno_t error);
59 |
60 | #define sl_default_handler ignore_handler_s
61 |
62 | #include "safe_mem_lib.h"
63 | #include "safe_str_lib.h"
64 |
65 | #ifdef __cplusplus
66 | }
67 | #endif
68 | #endif /* __SAFE_LIB_H__ */
69 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/include/safe_types.h:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * safe_types.h - C99 std types & defs or Linux kernel equivalents
3 | *
4 | * March 2007, Bo Berry
5 | * Modified 2012, Jonathan Toppins <jtoppins@users.sourceforge.net>
6 | *
7 | * Copyright (c) 2007-2013 by Cisco Systems, Inc
8 | * All rights reserved.
9 | *
10 | * Permission is hereby granted, free of charge, to any person
11 | * obtaining a copy of this software and associated documentation
12 | * files (the "Software"), to deal in the Software without
13 | * restriction, including without limitation the rights to use,
14 | * copy, modify, merge, publish, distribute, sublicense, and/or
15 | * sell copies of the Software, and to permit persons to whom the
16 | * Software is furnished to do so, subject to the following
17 | * conditions:
18 | *
19 | * The above copyright notice and this permission notice shall be
20 | * included in all copies or substantial portions of the Software.
21 | *
22 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
24 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
26 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
27 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
29 | * OTHER DEALINGS IN THE SOFTWARE.
30 | *------------------------------------------------------------------
31 | */
32 |
33 | #ifndef __SAFE_TYPES_H__
34 | #define __SAFE_TYPES_H__
35 |
36 | #ifdef __KERNEL__
37 | /* linux kernel environment */
38 |
39 | #include <linux/stddef.h>
40 | #include <linux/types.h>
41 | #include <linux/errno.h>
42 |
43 | /* errno_t isn't defined in the kernel */
44 | typedef int errno_t;
45 |
46 | #else
47 |
48 | #include <stdio.h>
49 |
50 | /* For systems without sys/types.h, prefer to get size_t from stdlib.h */
51 | /* some armcc environments don't have a sys/types.h in the environment */
52 | #ifdef _USE_STDLIB
53 | #include <stdlib.h>
54 | #include <ctype.h>
55 | #else
56 | #include <sys/types.h>
57 | #endif
58 |
59 | #include <inttypes.h>
60 | #include <stdint.h>
61 | #include <errno.h>
62 |
63 | typedef int errno_t;
64 |
65 | #include <stdbool.h>
66 |
67 | #endif /* __KERNEL__ */
68 | #endif /* __SAFE_TYPES_H__ */
69 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/include/safe_types.h.in:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * safe_types.h - C99 std types & defs or Linux kernel equivalents
3 | *
4 | * March 2007, Bo Berry
5 | * Modified 2012, Jonathan Toppins <jtoppins@users.sourceforge.net>
6 | *
7 | * Copyright (c) 2007-2013 by Cisco Systems, Inc
8 | * All rights reserved.
9 | *
10 | * Permission is hereby granted, free of charge, to any person
11 | * obtaining a copy of this software and associated documentation
12 | * files (the "Software"), to deal in the Software without
13 | * restriction, including without limitation the rights to use,
14 | * copy, modify, merge, publish, distribute, sublicense, and/or
15 | * sell copies of the Software, and to permit persons to whom the
16 | * Software is furnished to do so, subject to the following
17 | * conditions:
18 | *
19 | * The above copyright notice and this permission notice shall be
20 | * included in all copies or substantial portions of the Software.
21 | *
22 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
24 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
26 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
27 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
28 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
29 | * OTHER DEALINGS IN THE SOFTWARE.
30 | *------------------------------------------------------------------
31 | */
32 |
33 | #ifndef __SAFE_TYPES_H__
34 | #define __SAFE_TYPES_H__
35 |
36 | #ifdef __KERNEL__
37 | /* linux kernel environment */
38 |
39 | #include <linux/stddef.h>
40 | #include <linux/types.h>
41 | #include <linux/errno.h>
42 |
43 | /* errno_t isn't defined in the kernel */
44 | typedef int errno_t;
45 |
46 | #else
47 |
48 | #include <stdio.h>
49 | @INSERT_SYS_TYPES_H@
50 | @INSERT_INTTYPES_H@
51 | @INSERT_STDINT_H@
52 | @INSERT_ERRNO_H@
53 |
54 | @FALLBACK_ERRNO_T@
55 |
56 | @INSERT_BOOL_SUPPORT@
57 |
58 | #endif /* __KERNEL__ */
59 | #endif /* __SAFE_TYPES_H__ */
60 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/include/snprintf_s.h:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * sprintf_s.h -- Safe Sprintf Interfaces
3 | *
4 | * August 2014, D Wheeler
5 | *
6 | * Copyright (c) 2014 by Intel Corp
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 | #ifndef SPRINTF_S_H_
32 | #define SPRINTF_S_H_
33 |
34 | #include <safe_lib_errno.h>
35 |
36 |
37 | #define SNPRFNEGATE(x) (-1*(x))
38 |
39 |
40 |
41 | int snprintf_s_i(char *dest, rsize_t dmax, const char *format, int a);
42 | int snprintf_s_si(char *dest, rsize_t dmax, const char *format, char *s, int a);
43 | int snprintf_s_l(char *dest, rsize_t dmax, const char *format, long a);
44 | int snprintf_s_sl(char *dest, rsize_t dmax, const char *format, char *s, long a);
45 |
46 |
47 |
48 | #endif /* SPRINTF_S_H_ */
49 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/safeclib/abort_handler_s.c:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * abort_handler_s.c
3 | *
4 | * 2012, Jonathan Toppins <jtoppins@users.sourceforge.net>
5 | *
6 | * Copyright (c) 2012 Cisco Systems
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #include "safeclib_private.h"
33 |
34 | /**
35 | * NAME
36 | * abort_handler_s
37 | *
38 | * SYNOPSIS
39 | * #include "safe_lib.h"
40 | * void abort_handler_s(const char *msg, void *ptr, errno_t error)
41 | *
42 | * DESCRIPTION
43 | * This function writes a message on the standard error stream in
44 | * an implementation-defined format. The message shall include the
45 | * string pointed to by msg. The abort_handler_s function then calls
46 | * the abort function.
47 | *
48 | * SPECIFIED IN
49 | * ISO/IEC JTC1 SC22 WG14 N1172, Programming languages, environments
50 | * and system software interfaces, Extensions to the C Library,
51 | * Part I: Bounds-checking interfaces
52 | *
53 | * INPUT PARAMETERS
54 | * msg Pointer to the message describing the error
55 | *
56 | * ptr Pointer to aassociated data. Can be NULL.
57 | *
58 | * error The error code encountered.
59 | *
60 | * RETURN VALUE
61 | * Does not return to caller.
62 | *
63 | * ALSO SEE
64 | * ignore_handler_s()
65 | *
66 | */
67 |
68 | void abort_handler_s(const char *msg, void *ptr, errno_t error)
69 | {
70 | slprintf("ABORT CONSTRAINT HANDLER: (%u) %s\n", error,
71 | (msg) ? msg : "Null message");
72 | slabort();
73 | }
74 | EXPORT_SYMBOL(abort_handler_s)
75 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/safeclib/ignore_handler_s.c:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * ignore_handler_s.c
3 | *
4 | * 2012, Jonathan Toppins <jtoppins@users.sourceforge.net>
5 | *
6 | * Copyright (c) 2012 Cisco Systems
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #include "safeclib_private.h"
33 |
34 | /**
35 | * NAME
36 | * ignore_handler_s
37 | *
38 | * SYNOPSIS
39 | * #include "safe_lib.h"
40 | * void ignore_handler_s(const char *msg, void *ptr, errno_t error)
41 | *
42 | * DESCRIPTION
43 | * This function simply returns to the caller.
44 | *
45 | * SPECIFIED IN
46 | * ISO/IEC JTC1 SC22 WG14 N1172, Programming languages, environments
47 | * and system software interfaces, Extensions to the C Library,
48 | * Part I: Bounds-checking interfaces
49 | *
50 | * INPUT PARAMETERS
51 | * msg Pointer to the message describing the error
52 | *
53 | * ptr Pointer to aassociated data. Can be NULL.
54 | *
55 | * error The error code encountered.
56 | *
57 | * RETURN VALUE
58 | * Returns no value.
59 | *
60 | * ALSO SEE
61 | * abort_handler_s()
62 | *
63 | */
64 |
65 | void ignore_handler_s(const char *msg, void *ptr, errno_t error)
66 | {
67 |
68 | sldebug_printf("IGNORE CONSTRAINT HANDLER: (%u) %s\n", error,
69 | (msg) ? msg : "Null message");
70 | return;
71 | }
72 | EXPORT_SYMBOL(ignore_handler_s)
73 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/safeclib/mem_primitives_lib.h:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * mem_primitives_lib.h - Unguarded Memory Copy Routines
3 | *
4 | * October 2008, Bo Berry
5 | *
6 | * Copyright (c) 2008-2011 by Cisco Systems, Inc
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #ifndef __MEM_PRIMITIVES_LIB_H__
33 | #define __MEM_PRIMITIVES_LIB_H__
34 |
35 | #include "safeclib_private.h"
36 |
37 | /*
38 | * These are prototypes for _unguarded_ memory routines. The caller must
39 | * validate all parameters prior to invocation. Useful for diagnostics
40 | * and system initialization processing.
41 | */
42 |
43 | /* moves (handles overlap) memory */
44 | extern void
45 | mem_prim_move(void *dest, const void *src, uint32_t length);
46 |
47 |
48 | /* uint8_t moves (handles overlap) memory */
49 | extern void
50 | mem_prim_move8(uint8_t *dest, const uint8_t *src, uint32_t length);
51 |
52 | /* uint16_t moves (handles overlap) memory */
53 | extern void
54 | mem_prim_move16(uint16_t *dest, const uint16_t *src, uint32_t length);
55 |
56 | /* uint32_t moves (handles overlap) memory */
57 | extern void
58 | mem_prim_move32(uint32_t *dest, const uint32_t *src, uint32_t length);
59 |
60 |
61 | /* set bytes */
62 | extern void
63 | mem_prim_set(void *dest, uint32_t dmax, uint8_t value);
64 |
65 | /* set uint16_ts */
66 | extern void
67 | mem_prim_set16(uint16_t *dest, uint32_t dmax, uint16_t value);
68 |
69 | /* set uint32_ts */
70 | extern void
71 | mem_prim_set32(uint32_t *dest, uint32_t dmax, uint32_t value);
72 |
73 |
74 | #endif /* __MEM_PRIMITIVES_LIB_H__ */
75 |
--------------------------------------------------------------------------------
/columnar/vendor/safestringlib/safeclib/safe_mem_constraint.h:
--------------------------------------------------------------------------------
1 | /*------------------------------------------------------------------
2 | * safe_mem_constraint.h
3 | *
4 | * October 2008, Bo Berry
5 | *
6 | * Copyright (c) 2008, 2009 by Cisco Systems, Inc.
7 | * All rights reserved.
8 | *
9 | * Permission is hereby granted, free of charge, to any person
10 | * obtaining a copy of this software and associated documentation
11 | * files (the "Software"), to deal in the Software without
12 | * restriction, including without limitation the rights to use,
13 | * copy, modify, merge, publish, distribute, sublicense, and/or
14 | * sell copies of the Software, and to permit persons to whom the
15 | * Software is furnished to do so, subject to the following
16 | * conditions:
17 | *
18 | * The above copyright notice and this permission notice shall be
19 | * included in all copies or substantial portions of the Software.
20 | *
21 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 | * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 | * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 | * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 | * OTHER DEALINGS IN THE SOFTWARE.
29 | *------------------------------------------------------------------
30 | */
31 |
32 | #ifndef __SAFE_MEM_CONSTRAINT_H__
33 | #define __SAFE_MEM_CONSTRAINT_H__
34 |
35 | #include "safeclib_private.h"
36 |
37 | /*
38 | * Function used by the libraries to invoke the registered
39 | * runtime-constraint handler. Always needed.
40 | */
41 | extern void invoke_safe_mem_constraint_handler(
42 | const char *msg,
43 | void *ptr,
44 | errno_t error);
45 |
46 | #endif /* __SAFE_MEM_CONSTRAINT_H__ */
47 |
--------------------------------------------------------------------------------
/docker-bake.hcl:
--------------------------------------------------------------------------------
1 | variable "POSTGRES_REPO" {
2 | default = "ghcr.io/hydradatabase/hydra"
3 | }
4 |
5 | variable "SPILO_REPO" {
6 | default = "011789831835.dkr.ecr.us-east-1.amazonaws.com/spilo"
7 | }
8 |
9 | variable "SPILO_VERSION" {
10 | default = "3.2-p2"
11 | }
12 |
13 | variable "POSTGRES_BASE_VERSION" {
14 | default = "14"
15 | }
16 |
17 | variable "SPILO_POSTGRES_VERSION" {
18 | default = "14"
19 | }
20 |
21 | variable "SPILO_POSTGRES_OLD_VERSIONS" {
22 | default = "13"
23 | }
24 |
25 | group "default" {
26 | targets = ["postgres", "spilo"]
27 | }
28 |
29 | target "shared" {
30 | platforms = [
31 | "linux/amd64",
32 | "linux/arm64"
33 | ]
34 |
35 | args = {
36 | TIMESTAMP = "${timestamp()}"
37 | }
38 | }
39 |
40 | target "postgres" {
41 | inherits = ["shared"]
42 |
43 | contexts = {
44 | columnar = "target:columnar_${POSTGRES_BASE_VERSION}"
45 | postgres_base = "docker-image://postgres:${POSTGRES_BASE_VERSION}-bookworm"
46 | }
47 |
48 | args = {
49 | POSTGRES_BASE_VERSION = "${POSTGRES_BASE_VERSION}"
50 | }
51 |
52 | tags = [
53 | "${POSTGRES_REPO}:latest",
54 | "${POSTGRES_REPO}:${POSTGRES_BASE_VERSION}"
55 | ]
56 | }
57 |
58 | target "spilo" {
59 | inherits = ["shared"]
60 |
61 | dockerfile = "Dockerfile.spilo"
62 |
63 | contexts = {
64 | spilo_base = "target:spilo_base"
65 | columnar_13 = "target:columnar_13"
66 | columnar_14 = "target:columnar_14"
67 | columnar_15 = "target:columnar_15"
68 | }
69 |
70 | args = {
71 | POSTGRES_BASE_VERSION = "${SPILO_POSTGRES_VERSION}"
72 | }
73 |
74 | tags = [
75 | "${SPILO_REPO}:latest",
76 | "${SPILO_REPO}:${SPILO_VERSION}-latest"
77 | ]
78 | }
79 |
80 | target "spilo_base" {
81 | inherits = ["shared"]
82 |
83 | context = "https://github.com/zalando/spilo.git#${SPILO_VERSION}:postgres-appliance"
84 |
85 | args = {
86 | TIMESCALEDB = ""
87 | PGVERSION = "${SPILO_POSTGRES_VERSION}"
88 | PGOLDVERSIONS = "${SPILO_POSTGRES_OLD_VERSIONS}"
89 | }
90 | }
91 |
92 | target "columnar" {
93 | inherits = ["shared"]
94 | context = "columnar"
95 | target = "output"
96 | }
97 |
98 | target "columnar_13" {
99 | inherits = ["columnar"]
100 |
101 | args = {
102 | POSTGRES_BASE_VERSION = 13
103 | }
104 | }
105 |
106 | target "columnar_14" {
107 | inherits = ["columnar"]
108 |
109 | args = {
110 | POSTGRES_BASE_VERSION = 14
111 | }
112 | }
113 |
114 | target "columnar_15" {
115 | inherits = ["columnar"]
116 |
117 | args = {
118 | POSTGRES_BASE_VERSION = 15
119 | }
120 | }
121 |
122 | target "columnar_16" {
123 | inherits = ["columnar"]
124 |
125 | args = {
126 | POSTGRES_BASE_VERSION = 16
127 | }
128 | }
129 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | services:
2 | hydra:
3 | container_name: hydra
4 | image: ghcr.io/hydradatabase/hydra:latest
5 | ports:
6 | - ${POSTGRES_PORT}:5432
7 | environment:
8 | POSTGRES_USER: ${POSTGRES_USER}
9 | POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
10 | volumes:
11 | - ./volumes/db:/var/lib/postgresql/data
12 | - ./files/postgres/postgresql.conf:/etc/postgresql/postgresql.conf
13 | command: postgres -c 'config_file=/etc/postgresql/postgresql.conf'
14 |
--------------------------------------------------------------------------------
/files/postgres/docker-entrypoint-initdb.d/columnar.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -e
4 |
5 | psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
6 | CREATE EXTENSION IF NOT EXISTS columnar;
7 | ALTER EXTENSION columnar UPDATE;
8 | ALTER DATABASE "${POSTGRES_DB}" SET default_table_access_method = 'columnar';
9 | EOSQL
10 |
--------------------------------------------------------------------------------
/files/spilo/postgres-appliance/scripts/multicorn/after-create.sql:
--------------------------------------------------------------------------------
1 | GRANT USAGE ON FOREIGN DATA WRAPPER multicorn TO admin;
2 |
--------------------------------------------------------------------------------
/files/spilo/postgres-appliance/scripts/mysql_fdw/after-create.sql:
--------------------------------------------------------------------------------
1 | GRANT USAGE ON FOREIGN DATA WRAPPER mysql_fdw TO admin;
2 |
--------------------------------------------------------------------------------
/files/spilo/postgres-appliance/scripts/parquet_s3_fdw/after-create.sql:
--------------------------------------------------------------------------------
1 | GRANT USAGE ON FOREIGN DATA WRAPPER parquet_s3_fdw TO admin;
2 |
--------------------------------------------------------------------------------
/third-party/pgxman/pgxman_13.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | postgres:
3 | version: "13"
4 | extensions:
5 | - name: "multicorn"
6 | version: "2.4.0+b68b75c"
7 | - name: "mysql_fdw"
8 | version: "2.9.1"
9 | - name: "parquet_s3_fdw"
10 | version: "1.0.0+5298b7f"
11 | - name: "pg_ivm"
12 | version: "1.7.0"
13 | - name: "pgvector"
14 | version: "0.5.1"
15 | - name: "pg_hint_plan"
16 | version: "1.3.9"
17 |
--------------------------------------------------------------------------------
/third-party/pgxman/pgxman_13_spilo.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | postgres:
3 | version: "13"
4 | extensions:
5 | - name: "multicorn"
6 | version: "2.4.0+b68b75c"
7 | - name: "mysql_fdw"
8 | version: "2.9.1"
9 | - name: "parquet_s3_fdw"
10 | version: "1.0.0+5298b7f"
11 | - name: "pg_ivm"
12 | version: "1.7.0"
13 | - name: "pgvector"
14 | version: "0.5.1"
15 | overwrite: true
16 | - name: "pg_hint_plan"
17 | version: "1.3.9"
18 |
--------------------------------------------------------------------------------
/third-party/pgxman/pgxman_14.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | postgres:
3 | version: "14"
4 | extensions:
5 | - name: "multicorn"
6 | version: "2.4.0+b68b75c"
7 | - name: "mysql_fdw"
8 | version: "2.9.1"
9 | - name: "parquet_s3_fdw"
10 | version: "1.0.0+5298b7f"
11 | - name: "pg_ivm"
12 | version: "1.7.0"
13 | - name: "pgvector"
14 | version: "0.5.1"
15 | - name: "pg_hint_plan"
16 | version: "1.4.2"
17 | - name: "wrappers"
18 | version: "0.2.0"
19 | - name: "pgsodium"
20 | version: "3.1.9"
21 | - name: "supabase_vault"
22 | version: "0.2.9"
23 |
--------------------------------------------------------------------------------
/third-party/pgxman/pgxman_14_spilo.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | postgres:
3 | version: "14"
4 | extensions:
5 | - name: "multicorn"
6 | version: "2.4.0+b68b75c"
7 | overwrite: true
8 | - name: "mysql_fdw"
9 | version: "2.9.1"
10 | - name: "parquet_s3_fdw"
11 | version: "1.0.0+5298b7f"
12 | overwrite: true
13 | - name: "pg_ivm"
14 | version: "1.7.0"
15 | - name: "pgvector"
16 | version: "0.5.1"
17 | overwrite: true
18 | - name: "pg_hint_plan"
19 | version: "1.4.2"
20 | - name: "wrappers"
21 | version: "0.2.0"
22 | - name: "pgsodium"
23 | version: "3.1.9"
24 | - name: "supabase_vault"
25 | version: "0.2.9"
26 |
--------------------------------------------------------------------------------
/third-party/pgxman/pgxman_15.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | postgres:
3 | version: "15"
4 | extensions:
5 | - name: "multicorn"
6 | version: "2.4.0+b68b75c"
7 | - name: "mysql_fdw"
8 | version: "2.9.1"
9 | - name: "parquet_s3_fdw"
10 | version: "1.0.0+5298b7f"
11 | - name: "pg_ivm"
12 | version: "1.7.0"
13 | - name: "pgvector"
14 | version: "0.5.1"
15 | - name: "pg_hint_plan"
16 | version: "1.5.1"
17 | - name: "wrappers"
18 | version: "0.2.0"
19 | - name: "pgsodium"
20 | version: "3.1.9"
21 | - name: "supabase_vault"
22 | version: "0.2.9"
23 |
--------------------------------------------------------------------------------
/third-party/pgxman/pgxman_16.yaml:
--------------------------------------------------------------------------------
1 | apiVersion: v1
2 | postgres:
3 | version: "16"
4 | extensions:
5 | - name: "mysql_fdw"
6 | version: "2.9.1"
7 | - name: "pg_ivm"
8 | version: "1.7.0"
9 | - name: "pgvector"
10 | version: "0.5.1"
11 | - name: "pg_hint_plan"
12 | version: "1.6.0"
13 | - name: "wrappers"
14 | version: "0.2.0"
15 | - name: "pgsodium"
16 | version: "3.1.9"
17 | - name: "supabase_vault"
18 | version: "0.2.9"
19 |
--------------------------------------------------------------------------------