├── .gitlab-ci.yml ├── Dockerfile ├── LICENSE ├── README.md ├── test.sh └── test.sql /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | image: docker:latest 2 | services: 3 | - docker:dind 4 | 5 | stages: 6 | - test 7 | - build 8 | 9 | testing_image: 10 | stage: test 11 | script: 12 | - docker build -t pg-350d . 13 | - ./test.sh 14 | 15 | upload_image: 16 | stage: build 17 | variables: 18 | IMAGE_TAG: $CI_REGISTRY_IMAGE:15.3 19 | script: 20 | - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY 21 | - docker build -t $IMAGE_TAG . 22 | - docker push $IMAGE_TAG 23 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM postgres:15.4 2 | MAINTAINER Olivier El Mekki 3 | 4 | RUN apt update && apt install -y build-essential curl postgresql-server-dev-15 5 | RUN curl https://ftp.postgresql.org/pub/source/v15.4/postgresql-15.4.tar.bz2 -o /postgresql-15.4.tar.bz2 6 | RUN cd / && tar xvf postgresql-15.4.tar.bz2 7 | RUN cd /postgresql-15.4/contrib/cube && sed -i 's/#define CUBE_MAX_DIM (100)/#define CUBE_MAX_DIM (350)/' cubedata.h && \ 8 | USE_PGXS=true make && USE_PGXS=true make install 9 | RUN echo -e "\nen_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This project is in public domain. 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pg350d 2 | 3 | Docker build of postgresql-15.3 changing the dimension limit for the cube 4 | extension, raising it to 350. 5 | 6 | This is needed to be able to work with words embedding or other machine 7 | learning related vectors with postgres. 8 | 9 | > Note: since pg350d was released, there has been some efforts to support 10 | > machine learning friendly vectors in [pgvector](https://github.com/pgvector/pgvector), 11 | > which supports up to 16k dimensions. 12 | 13 | You can easily generate a build for your own need in term of dimensions by 14 | editing this dockerfile. 15 | 16 | ## What is the problem again? 17 | 18 | The cube extension, which you'll use to perform operations on vectors, has 19 | a hard limit of 100 dimensions per vector. 20 | 21 | ## Download 22 | 23 | A built image is available on Gitlab's registry. You can pull it this way: 24 | 25 | ``` 26 | docker pull registry.gitlab.com/oelmekki/postgres-350d:15.3 27 | ``` 28 | 29 | ## Is it safe? 30 | 31 | Patching the hardcoded limit is [the recommended way in postgres 32 | doc](https://www.postgresql.org/docs/current/cube.html#id-1.11.7.20.9). 33 | 34 | I've been using it for a few years in production, and didn't encounter any 35 | problem. 36 | 37 | ## How to raise postgresql's cube extension dimensions limit? 38 | 39 | Even if you're not familiar with docker, reading the Dockerfile from this repos 40 | should be easy enough to teach you how to do it yourself. Spoiler: 41 | 42 | * sources are in the `contrib/cube/` directory from postgres sources 43 | * limit is in `contrib/cube/cubedata.h` 44 | * make and make install should be ran with the `USE_PGXS=true` env variable 45 | 46 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | ##!/usr/bin/env bash 2 | 3 | echo "Starting database…" 4 | ID=$(docker run --rm -e POSTGRES_HOST_AUTH_METHOD=trust -d pg-350d) 5 | sleep 10 6 | 7 | echo "Running test…" 8 | IP=$(docker inspect $ID | grep '"IPAddress"' | head -n 1 | awk '{ print $2 }' | sed 's/[",]//g') 9 | psql -U postgres -h $IP -f ./test.sql | grep "ERROR" 10 | ERR=$(test "$?" != "1") 11 | 12 | if [[ -n "$ERR" ]]; then 13 | echo "$ERR" 14 | else 15 | echo "Success." 16 | fi 17 | 18 | docker stop $ID &> /dev/null 19 | exit $ERR 20 | -------------------------------------------------------------------------------- /test.sql: -------------------------------------------------------------------------------- 1 | CREATE EXTENSION cube; 2 | CREATE TABLE vectors(vector cube); 3 | -- vector of 350 dimensions 4 | INSERT INTO vectors(vector) VALUES(cube(ARRAY[ 5 | 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1 6 | ])); 7 | --------------------------------------------------------------------------------