├── ntpd.sv.conf ├── sshd.sv.conf ├── gdbserverd.sv.conf ├── supervisord.conf ├── gdbserverd ├── verticad.sv.conf ├── docker-compose.yml ├── Dockerfile ├── verticad └── README.md /ntpd.sv.conf: -------------------------------------------------------------------------------- 1 | [program:ntpd] 2 | command=/usr/sbin/ntpd -n 3 | autostart=true 4 | autorestart=true 5 | 6 | -------------------------------------------------------------------------------- /sshd.sv.conf: -------------------------------------------------------------------------------- 1 | [program:sshd] 2 | command=/usr/sbin/sshd -D 3 | autostart=true 4 | autorestart=true 5 | 6 | -------------------------------------------------------------------------------- /gdbserverd.sv.conf: -------------------------------------------------------------------------------- 1 | [eventlistener:gdbserverd] 2 | command=/bin/bash -c "exec /usr/local/bin/gdbserverd" 3 | events=PROCESS_COMMUNICATION_STDOUT 4 | numprocs=1 -------------------------------------------------------------------------------- /supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | user=root 4 | pidfile=/var/run/supervisord.pid 5 | loglevel=critical 6 | 7 | [include] 8 | files = /etc/supervisor/conf.d/*.conf 9 | 10 | -------------------------------------------------------------------------------- /gdbserverd: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | printf "READY\n" 4 | while read line; do 5 | # Could be using gdbserver :${GDBSERVER_PORT} --attach $(pgrep -x vertica) >&2 if attached to the running PID 6 | gdbserver --multi :${GDBSERVER_PORT} >&2 7 | printf "RESULT 2\nOK" 8 | done < /dev/stdin -------------------------------------------------------------------------------- /verticad.sv.conf: -------------------------------------------------------------------------------- 1 | [program:verticad] 2 | command=/bin/bash -c "exec /usr/local/bin/verticad -DFOREGROUND" 3 | stopsignal=INT 4 | stopwaitsecs=300 5 | autostart=true 6 | autorestart=false 7 | startsecs=0 8 | startretries=0 9 | stdout_logfile=/dev/stdout 10 | stdout_capture_maxbytes=1MB 11 | stdout_logfile_maxbytes=0 12 | redirect_stderr=true 13 | 14 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | node_1: 4 | build: 5 | context: . 6 | ports: 7 | - "5433:5433" 8 | environment: 9 | NODE_TYPE: "master" 10 | CLUSTER_NODES: "node_1,node_2,node_3" 11 | WITH_VMART: 'false' 12 | ENABLE_WATCHDOG: 'true' 13 | depends_on: 14 | - node_2 15 | - node_3 16 | links: 17 | - node_2 18 | - node_3 19 | cap_add: 20 | - SYS_NICE 21 | - SYS_RESOURCE 22 | - SYS_PTRACE 23 | volumes: 24 | - node_1_data:/opt/vertica 25 | - ${VERTICA_RPM_PATH}:/tmp/vertica.rpm 26 | node_2: 27 | build: 28 | context: . 29 | environment: 30 | NODE_TYPE: "slave" 31 | VERTICA_HOME: "/opt/vertica" 32 | expose: 33 | - "22" 34 | - "5433" 35 | cap_add: 36 | - SYS_NICE 37 | - SYS_RESOURCE 38 | - SYS_PTRACE 39 | volumes: 40 | - node_2_data:/opt/vertica 41 | - ${VERTICA_RPM_PATH}:/tmp/vertica.rpm 42 | node_3: 43 | build: 44 | context: . 45 | environment: 46 | NODE_TYPE: "slave" 47 | VERTICA_HOME: "/opt/vertica" 48 | expose: 49 | - "22" 50 | - "5433" 51 | cap_add: 52 | - SYS_NICE 53 | - SYS_RESOURCE 54 | - SYS_PTRACE 55 | volumes: 56 | - node_3_data:/opt/vertica 57 | - ${VERTICA_RPM_PATH}:/tmp/vertica.rpm 58 | volumes: 59 | node_1_data: 60 | driver: local 61 | node_2_data: 62 | driver: local 63 | node_3_data: 64 | driver: local 65 | 66 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM centos:centos7 2 | MAINTAINER Francois Jehl 3 | 4 | # Environment Variables 5 | ENV VERTICA_HOME=/opt/vertica \ 6 | WITH_VMART=false \ 7 | NODE_TYPE=master \ 8 | CLUSTER_NODES=localhost \ 9 | GDBSERVER_PORT=2159 \ 10 | ENABLE_WATCHDOG=false 11 | ARG ENABLE_GDB_DEBUG=true 12 | 13 | RUN localedef -i en_US -f UTF-8 en_US.UTF-8 14 | 15 | # Yum dependencies 16 | RUN yum install -y \ 17 | which \ 18 | openssh-server \ 19 | openssh-clients \ 20 | openssl \ 21 | iproute \ 22 | dialog \ 23 | gdb \ 24 | gdb-gdbserver \ 25 | sysstat \ 26 | mcelog \ 27 | bc \ 28 | ntp \ 29 | gcc-c++ \ 30 | cmake \ 31 | python-setuptools 32 | 33 | # Debug infos for GDB 34 | RUN if [ ${ENABLE_GDB_DEBUG} = 'true' ]; then debuginfo-install -y \ 35 | expat \ 36 | glibc \ 37 | keyutils-libs \ 38 | libcom_err \ 39 | libgcc \ 40 | libstdc++ \ 41 | zlib \ 42 | ; fi 43 | 44 | # Install supervisor 45 | RUN easy_install supervisor 46 | 47 | # DBAdmin account configuration 48 | RUN groupadd -r verticadba \ 49 | && useradd -r -m -g verticadba dbadmin 50 | 51 | USER dbadmin 52 | RUN echo "export LANG=en_US.UTF-8" >> ~/.bash_profile \ 53 | && echo "export TZ=/usr/share/zoneinfo/Etc/Universal" >> ~/.bash_profile \ 54 | && mkdir ~/.ssh && cd ~/.ssh && ssh-keygen -t rsa -q -f id_rsa \ 55 | && cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys 56 | 57 | # Root SSH configuration 58 | USER root 59 | RUN mkdir ~/.ssh && cd ~/.ssh && ssh-keygen -t rsa -q -f id_rsa \ 60 | && cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys \ 61 | && /usr/bin/ssh-keygen -A 62 | 63 | # Vertica specific system requirements 64 | RUN echo "session required pam_limits.so" >> /etc/pam.d/su \ 65 | && echo "dbadmin - nofile 65536" >> /etc/security/limits.conf \ 66 | && echo "dbadmin - nice 0" >> /etc/security/limits.conf 67 | 68 | #SupervisorD configuration 69 | COPY sshd.sv.conf ntpd.sv.conf verticad.sv.conf gdbserverd.sv.conf /etc/supervisor/conf.d/ 70 | COPY supervisord.conf /etc/supervisord.conf 71 | 72 | # Vertica and GDB daemon-like startup scripts 73 | COPY verticad /usr/local/bin/verticad 74 | COPY gdbserverd /usr/local/bin/gdbserverd 75 | 76 | EXPOSE 5433 77 | EXPOSE ${GDBSERVER_PORT} 78 | #Starting supervisor 79 | CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisord.conf"] 80 | -------------------------------------------------------------------------------- /verticad: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # SIGINT trap to shut down the DB properly upon CTRL+C 4 | at_sigint() { 5 | if [[ "$NODE_TYPE" == "master" ]] 6 | then 7 | echo "Received SIGINT on the master node: stopping the database..." 8 | su - dbadmin -c "${VERTICA_HOME}/bin/admintools -t stop_db -d docker --force" 9 | echo "Database has been stopped on the master node". 10 | else 11 | while ! su - dbadmin -c "${VERTICA_HOME}/bin/admintools -t db_status -s DOWN" | grep -q docker && pgrep --exact vertica >/dev/null 12 | do 13 | echo "Waiting for the database to shutdown before stopping the node..." 14 | sleep 2 15 | done 16 | echo "Database not running anymore: shutting down the node." 17 | fi 18 | exit 0 19 | } 20 | 21 | cluster_is_up() { 22 | su - dbadmin -c "${VERTICA_HOME}/bin/admintools -t db_status -s UP" | grep -q docker 23 | } 24 | 25 | start_cluster() { 26 | su - dbadmin -c "${VERTICA_HOME}/bin/admintools -t start_db -d docker --force --noprompts" 27 | } 28 | 29 | get_vertica_pid() { 30 | pgrep -x vertica 31 | } 32 | 33 | nb_down_nodes() { 34 | su - dbadmin -c "/opt/vertica/bin/admintools -t view_cluster -x | grep docker | grep -v UP | wc -l" 35 | } 36 | 37 | restart_down_nodes() { 38 | su - dbadmin -c "/opt/vertica/bin/admintools -t view_cluster -x | awk '/docker/ && ! /UP/{print \$3}' | paste -s -d, - | xargs /opt/vertica/bin/admintools -t restart_node -d docker -s " 39 | } 40 | 41 | print_info_message() { 42 | echo "---------------------------------------------------------------------------------------------------------------------------------------" 43 | cat $1 44 | echo "---------------------------------------------------------------------------------------------------------------------------------------" 45 | } 46 | 47 | set -e 48 | 49 | if [[ ! -d "${VERTICA_HOME}" ]] 50 | then 51 | echo "${VERTICA_HOME} does not exist in the guest. Please mount a data volume on ${VERTICA_HOME} and restart the guest." 52 | exit 1 53 | fi 54 | 55 | capabilities=$(capsh --print | grep Current) 56 | for capability in SYS_NICE, SYS_RESOURCE 57 | do 58 | if ! echo $capabilities | grep -qi $capability 59 | then 60 | echo "Missing $capability. Start the container with --cap-add $capability." 61 | exit 1 62 | fi 63 | done 64 | 65 | 66 | if [[ ! -d "${VERTICA_HOME}/catalog" ]] || [[ ! -d "${VERTICA_HOME}/data" ]] 67 | then 68 | echo "Creating data and catalog directories..." 69 | chown -R dbadmin:verticadba ${VERTICA_HOME} 70 | su - dbadmin -c "mkdir -p ${VERTICA_HOME}/catalog -m 0755" 71 | su - dbadmin -c "mkdir -p ${VERTICA_HOME}/data -m 0755" 72 | fi 73 | echo "Data and catalog dirs exist on this node." 74 | 75 | # Everything that gets executed in this block should happen once and only once 76 | # Hence the concept of 'master' node, that has no meaning in Vertica itself 77 | if [[ "$NODE_TYPE" == "master" ]] 78 | then 79 | if [[ ! -d "${VERTICA_HOME}/bin" ]] 80 | then 81 | echo "Installing RPM on this node..." 82 | rpm -Uvh /tmp/vertica.rpm 83 | chown -R dbadmin:verticadba ${VERTICA_HOME}/config 84 | chown -R dbadmin:verticadba ${VERTICA_HOME}/log 85 | fi 86 | echo "The RPM is installed." 87 | 88 | if [[ ! -e ${VERTICA_HOME}/config/admintools.conf ]] 89 | then 90 | LICENSE="CE" 91 | if [[ -f "/tmp/license.dat" ]] 92 | then 93 | LICENSE="/tmp/license.dat" 94 | fi 95 | 96 | echo "Setting up a Vertica cluster from this master node... License : $LICENSE" 97 | 98 | INSTALL_COMMAND="${VERTICA_HOME}/sbin/install_vertica \ 99 | --hosts "$CLUSTER_NODES" \ 100 | --rpm /tmp/vertica.rpm \ 101 | --no-system-configuration \ 102 | --license "$LICENSE" \ 103 | --accept-eula \ 104 | --dba-user dbadmin \ 105 | --dba-user-password-disabled \ 106 | --failure-threshold NONE \ 107 | --point-to-point \ 108 | --ignore-aws-instance-type" 109 | 110 | if [[ ! -z "$VERTICA_LARGE_CLUSTER" ]] 111 | then 112 | INSTALL_COMMAND="$INSTALL_COMMAND --large-cluster $VERTICA_LARGE_CLUSTER" 113 | fi 114 | 115 | echo "RUNNING $INSTALL_COMMAND" 116 | eval $INSTALL_COMMAND 117 | fi 118 | echo "The cluster is set up." 119 | 120 | # Sets up a cluster (a set of nodes sharing the same spread configuration) 121 | if ! su - dbadmin -c "${VERTICA_HOME}/bin/admintools -t view_cluster" | grep -q docker 122 | then 123 | echo "Now creating the database..." 124 | su - dbadmin -c "${VERTICA_HOME}/bin/admintools \ 125 | -t create_db \ 126 | -s "$CLUSTER_NODES" \ 127 | -d docker \ 128 | -c ${VERTICA_HOME}/catalog \ 129 | -D ${VERTICA_HOME}/data \ 130 | --skip-fs-checks" 131 | fi 132 | echo "The docker database has been created on the cluster." 133 | 134 | # Starts the cluster if not already started 135 | echo "Checking cluster status..." 136 | if ! cluster_is_up 137 | then 138 | echo "Starting Vertica..." 139 | start_cluster 140 | fi 141 | echo "Vertica is started." 142 | 143 | # Installs the VMart test schema if this has been supplied as environment variable 144 | if [ ${WITH_VMART} = 'true' ] 145 | then 146 | if ! su - dbadmin -c "${VERTICA_HOME}/bin/vsql -qt -c 'select schema_name from schemata'" | grep -q online_sales 147 | then 148 | echo "Importing VMart schema data in this cluster" 149 | su - dbadmin -c "cd /opt/vertica/examples/VMart_Schema/;./vmart_gen >/dev/null 2>&1;/opt/vertica/bin/vsql -q -t -f vmart_define_schema.sql >/dev/null 2>&1;/opt/vertica/bin/vsql -q -t -f vmart_load_data.sql >/dev/null 2>&1" 150 | fi 151 | echo "The VMart schema is imported." 152 | fi 153 | 154 | ip=$(hostname --all-ip-addresses | awk '{print $1}') 155 | print_info_message <VERTICASTART" 163 | 164 | # Registers a SIGINT trap and sleeps until CTRL+C is pressed 165 | trap at_sigint INT 166 | while true; do 167 | # If the database should be restarted automatically, check for the status every 5 sec 168 | if [ "${NODE_TYPE}" = "master" ] && [ "${ENABLE_WATCHDOG}" = 'true' ] 169 | then 170 | if ! cluster_is_up 171 | then 172 | echo "The whole cluster is down! Restarting..." 173 | start_cluster && print_info_message <(echo "You can attach to the new Vertica PID using \"attach $(get_vertica_pid) \".") 174 | fi 175 | nb_down=$(nb_down_nodes) 176 | if cluster_is_up && [ $nb_down -gt 0 ] 177 | then 178 | echo "$nb_down node(s) down! Restarting..." 179 | restart_down_nodes && print_info_message <(echo "You can attach to the new Vertica PID using \"attach $(get_vertica_pid) \".") 180 | fi 181 | fi 182 | sleep 5 183 | done 184 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # docker-vertica 2 | ![Build Config](https://img.shields.io/docker/cloud/automated/fjehl/docker-vertica.svg) 3 | ![Build Status](https://img.shields.io/docker/cloud/build/fjehl/docker-vertica.svg) 4 | 5 | A docker image for Vertica Community, run as single node or multi-node. 6 | Can be found on DockerHub at . 7 | It can be used as a sandbox environment for UDF debugging. 8 | 9 | ## Table of Contents 10 | 1. [Common Usage](#common-usage) 11 | * [Extend it in another Dockerfile](#extend-it-in-another-dockerfile) 12 | * [Start a single-node cluster](#start-a-single-node-cluster) 13 | * [Start a multi-node cluster](#start-a-multi-node-cluster) 14 | 2. [UDF Development](#udf-development) 15 | 3. [Advanced host configuration](#advanced-host-configuration) 16 | 17 | 18 | ## Common usage 19 | This repository provides a Dockerfile and docker-compose.yml that permit starting Vertica respectively: 20 | - In standalone / single-node mode using **Docker** 21 | - In cluster mode using **Docker-Compose** 22 | In the two modes, Vertica expects an external volume to be mounted on its **/opt/vertica** which means data is persisted accross container runs. 23 | When started with docker-compose, no configuration is needed on the workstation, everything's declared in the docker-compose.yml file. In pure docker mode, the run command is a little bit more complicated and documented in the following section. 24 | 25 | ### Extend it in another Dockerfile 26 | You can of course derive an image from this one. 27 | Use the FROM docker directive from within your Dockerfile 28 | ``` 29 | FROM fjehl/docker-vertica:latest 30 | ``` 31 | 32 | Vertica is started using a verticad [http://supervisord.org/index.html](Supervisor) daemon, that emits a PROCESS_COMMUNICATION_STDOUT event on his stdout file descriptor. You can catch it using event handlers (see Supervisor documentation for this). Basically, your event handler should just send on its stdout a READY command once started, read lines from stdin, and issue a RESULT 2\\nOK once finished, as in the Python script you'll find in the following [http://supervisord.org/events.html](page). 33 | Nevertheless, you child image should 34 | - Add new supervisord programs or event handlers in /etc/supervisor/conf.d (they will get auto-loaded by the default supervisor.conf) 35 | - Have an entrypoint that runs supervisor like the current image 36 | 37 | ``` 38 | CMD ["/usr/bin/supervisord", "-n"] 39 | ``` 40 | ### Start a single-node cluster 41 | To build and run a single-node cluster, you can use standard docker commands. 42 | 43 | #### Building the image 44 | Use docker build inside the image directory. 45 | 46 | ``` 47 | docker build -t fjehl/docker-vertica . 48 | ``` 49 | 50 | #### Starting the container 51 | If you only need a single node cluster, you can use Docker run to run the image. 52 | Given that the RPM is installed at runtime, you need to download Vertica community edition at [http://my.vertica.com](http://my.vertica.com), and store it somewhere: you need to provide it to the container. 53 | You also need to provide a directory mounted as /opt/vertica in the guest. It can be a named volume or any other location on the host. 54 | Don't forget to add the SYS_NICE and SYS_RESOURCE capacities, otherwise the startup script will fail starting Vertica. 55 | 56 | ``` 57 | docker run \ 58 | -v ~/Downloads/vertica-8.0.0-0.x86_64.RHEL6.rpm:/tmp/vertica.rpm \ 59 | -v docker-vertica:/opt/vertica \ 60 | --cap-add SYS_NICE --cap-add SYS_RESOURCE --cap-add SYS_PTRACE\ 61 | --name docker-vertica \ 62 | -ti fjehl/docker-vertica 63 | ``` 64 | 65 | #### Killing the container in a clean way 66 | The container, and especially the vertica startup script (named **verticad**) are designed to handle a SIGINT signal, that will cleanly shutdown Vertica and prevent data corruption. 67 | ``` 68 | docker kill --signal SIGINT docker-vertica 69 | ``` 70 | 71 | #### Using your own license 72 | The Community Edition license only supports up to 3 nodes. If you ever want to go above this limit on your docker based cluster, you'll need an actual license. To get a license, contact the Vertica support team. 73 | If you have such license, you can use it simply by mounting in the container in `/tmp/license.dat`. For instance in `docker run` add the volume option : `-v /path/to/your/license:/tmp/license.dat` 74 | 75 | For docker-compose, edit the provided `docker-compose.yml` and add it to the volumes of the first container. 76 | 77 | ### Start a multi-node cluster 78 | A docker-compose.yml has been designed to ease configuration of a multi-node cluster. 79 | 80 | #### Download a Vertica RPM, expose it through an env variable 81 | 82 | Go to [the MyVertica website](http://my.vertica.com) and download a CentOS / RHEL version. 83 | Store it somewhere on your system, then export its location to an environment variable: 84 | 85 | ``` 86 | export VERTICA_RPM_PATH=~/Downloads/vertica-X.Y.Z-T.x86_64.RHEL6.rpm 87 | ``` 88 | 89 | #### Building the images 90 | Use docker-compose build inside the image directory. 91 | 92 | ``` 93 | docker-compose build 94 | ``` 95 | 96 | #### Starting the cluster 97 | The cluster is now ready to start. Just submit the "up" command. 98 | 99 | ``` 100 | docker-compose up 101 | ``` 102 | 103 | #### Killing the container in a clean way 104 | The container, and especially the vertica startup script (named **verticad**) are designed to handle a SIGINT signal, that will cleanly shutdown Vertica and prevent data corruption. 105 | 106 | ``` 107 | docker-compose kill -s SIGINT 108 | ``` 109 | 110 | #### Large cluster setup 111 | You may want to use a docker cluster to experiment how Vertica behaves in large cluster mode. You can enable that by adding an environment variable `VERTICA_LARGE_CLUSTER` and setting its value to the number of control nodes you want. 112 | You only have to set it to the first container in the `docker-compose.yml` file. 113 | 114 | 115 | ## UDF Development 116 | 117 | This container already contains all the useful tools to debug custom UDFs. (GDB and all available symbols). 118 | 119 | The following debug examples use [the simple GET_DATA_FROM_NODE() UDF](https://github.com/francoisjehl/getdatafromnode). 120 | 121 | ### Start the container with your code mounted 122 | 123 | Options are usually the same. You'll just supply a mount point that holds the build binaries. 124 | 125 | ``` 126 | docker run \ 127 | -v ~/Downloads/vertica-8.0.0-0.x86_64.RHEL6.rpm:/tmp/vertica.rpm \ 128 | -v docker-vertica:/opt/vertica \ 129 | -v /home/fjehl/git/vertica-getdatafromnode:/home/fjehl/git/vertica-getdatafromnode \ 130 | --cap-add SYS_NICE --cap-add SYS_RESOURCE --cap-add SYS_PTRACE \ 131 | --name docker-vertica \ 132 | -ti fjehl/docker-vertica 133 | ``` 134 | 135 | ### Compile the UDF inside the container 136 | 137 | Use your favourite build tool with a docker exec command. 138 | If you're using CMake, an example could look like this: 139 | 140 | ``` 141 | docker exec -ti docker-vertica /bin/bash -c ' 142 | rm -rf /home/dbadmin/lib/build && 143 | mkdir /home/dbadmin/lib/build && 144 | cd "$_" && 145 | cmake .. && 146 | make' 147 | ``` 148 | 149 | And would produce the following output: 150 | ``` 151 | -- The C compiler identification is GNU 4.8.5 152 | -- The CXX compiler identification is GNU 4.8.5 153 | -- Check for working C compiler: /usr/bin/cc 154 | -- Check for working C compiler: /usr/bin/cc -- works 155 | -- Detecting C compiler ABI info 156 | -- Detecting C compiler ABI info - done 157 | -- Check for working CXX compiler: /usr/bin/c++ 158 | -- Check for working CXX compiler: /usr/bin/c++ -- works 159 | -- Detecting CXX compiler ABI info 160 | -- Detecting CXX compiler ABI info - done 161 | -- Configuring done 162 | -- Generating done 163 | -- Build files have been written to: /home/dbadmin/lib/build 164 | Scanning dependencies of target getdatafromnode 165 | [ 50%] Building CXX object CMakeFiles/getdatafromnode.dir/src/GetDataFromNode.cpp.o 166 | [100%] Building CXX object CMakeFiles/getdatafromnode.dir/opt/vertica/sdk/include/Vertica.cpp.o 167 | Linking CXX shared library libgetdatafromnode.so 168 | [100%] Built target getdatafromnode 169 | ``` 170 | 171 | There's now a binary, it can be registered. 172 | 173 | ### Register it in Vertica 174 | 175 | You can directly connect from VSQL. Normally, the install script should have written the IP on stdout. 176 | It could be 172.17.0.2, as in the following example. 177 | Connect first to VSQL: 178 | 179 | ``` 180 | vsql -h 172.17.0.2 -U dbadmin 181 | ``` 182 | Then register the library: 183 | 184 | ``` 185 | CREATE OR REPLACE LIBRARY libgetdatafromnode 186 | AS '/home/fjehl/git/vertica-getdatafromnode/build/libgetdatafromnode.so'; 187 | ``` 188 | And the functions you want to use. 189 | 190 | ``` 191 | CREATE OR REPLACE TRANSFORM FUNCTION GET_DATA_FROM_NODE 192 | AS LANGUAGE 'C++' 193 | NAME 'GetDataFromNodeFactory' 194 | LIBRARY libgetdatafromnode 195 | NOT FENCED; 196 | ``` 197 | 198 | Test that everything seems fine: 199 | 200 | ``` 201 | SELECT 202 | GET_DATA_FROM_NODE(* USING PARAMETERS node='v_docker_node0001') OVER (PARTITION AUTO) 203 | FROM public.foo; 204 | ``` 205 | ``` 206 | bar 207 | ----- 208 | 1 209 | 2 210 | 3 211 | 4 212 | (4 rows) 213 | ``` 214 | ### Debug it using GDB 215 | 216 | Start GDB attached to the running GDBServer, as output by docker run's stdout. 217 | 218 | ``` 219 | (gdb) target extended-remote 172.17.0.2:2159 220 | ``` 221 | 222 | Then attach to Vertica's PID, again as advertised by docker run's stdout. 223 | 224 | ``` 225 | (gdb) attach 116 226 | ``` 227 | 228 | Symbols start to load: 229 | ``` 230 | GNU gdb (GDB) Red Hat Enterprise Linux 7.6.1-94.el7 231 | Copyright (C) 2013 Free Software Foundation, Inc. 232 | License GPLv3+: GNU GPL version 3 or later 233 | This is free software: you are free to change and redistribute it. 234 | There is NO WARRANTY, to the extent permitted by law. Type "show copying" 235 | and "show warranty" for details. 236 | This GDB was configured as "x86_64-redhat-linux-gnu". 237 | For bug reporting instructions, please see: 238 | . 239 | Attaching to process 114 240 | Reading symbols from /opt/vertica/bin/vertica...(no debugging symbols found)...done. 241 | Reading symbols from /opt/vertica/lib/libgssapi_krb5.so.2...(no debugging symbols found)...done. 242 | Loaded symbols for /opt/vertica/lib/libgssapi_krb5.so.2 243 | Reading symbols from /opt/vertica/lib/libkrb5.so.3...(no debugging symbols found)...done. 244 | Loaded symbols for /opt/vertica/lib/libkrb5.so.3 245 | Reading symbols from /opt/vertica/lib/libkrb5support.so.0...(no debugging symbols found)...done. 246 | Loaded symbols for /opt/vertica/lib/libkrb5support.so.0 247 | Reading symbols from /opt/vertica/lib/libk5crypto.so.3...(no debugging symbols found)...done. 248 | Loaded symbols for /opt/vertica/lib/libk5crypto.so.3 249 | Reading symbols from /opt/vertica/lib/libcom_err.so.3...(no debugging symbols found)...done. 250 | ``` 251 | 252 | And you finally get the GDB prompt. 253 | 254 | ``` 255 | (gdb) 256 | ``` 257 | 258 | You can now register code, and set some breakpoints for example: 259 | 260 | ``` 261 | (gdb) directory /home/fjehl/git/vertica-getdatafromnode/src/ 262 | (gdb) break /home/fjehl/git/vertica-getdatafromnode/src/GetDataFromNode.cpp:40 263 | ``` 264 | 265 | ``` 266 | Breakpoint 1 at 0x7f39782ebee0: file /home/fjehl/git/vertica-getdatafromnode/src/GetDataFromNode.cpp, line 40. 267 | ``` 268 | 269 | And restart the execution: 270 | ``` 271 | (gdb) c 272 | ``` 273 | 274 | The execution continues, as expected: 275 | 276 | 277 | ``` 278 | [New Thread 0x7f5b9ab2f700 (LWP 16618)] 279 | [New Thread 0x7f5b9cb33700 (LWP 16619)] 280 | [New Thread 0x7f5b94b0c700 (LWP 16620)] 281 | [New Thread 0x7f5b9c332700 (LWP 16621)] 282 | [New Thread 0x7f5b994db700 (LWP 16622)] 283 | [New Thread 0x7f5b98cda700 (LWP 16623)] 284 | [New Thread 0x7f5b9630f700 (LWP 16624)] 285 | [New Thread 0x7f5b95b0e700 (LWP 16625)] 286 | [New Thread 0x7f5b9530d700 (LWP 16626)] 287 | [New Thread 0x7f5b67fff700 (LWP 16627)] 288 | ``` 289 | 290 | Run the query within another VSQL. 291 | The breakpoint fires: 292 | 293 | ``` 294 | [Switching to Thread 0x7f5b9630f700 (LWP 16624)] 295 | 296 | Breakpoint 1, GetDataFromNode::processPartition (this=, srvInterface=..., inputReader=..., outputWriter=...) at /home/dbadmin/lib/src/GetDataFromNode.cpp:40 297 | 40 for (std::vector::iterator i = argCols.begin(); i < argCols.end(); i++) 298 | ``` 299 | 300 | You can now use all the popular GDB commands and features (n, step, bt, etc...) to debug your execution flow. 301 | You can follow this [excellent cheat sheet](http://darkdust.net/files/GDB%20Cheat%20Sheet.pdf) if you never used GDB. 302 | 303 | ### Debug it in Visual Studio Code 304 | 305 | You can use the following configuration. Edit the destination host, path and source files with yours. 306 | 307 | 308 | ``` 309 | { 310 | "version": "0.2.0", 311 | "configurations": [ 312 | { 313 | "name": "Launch", 314 | "type": "cppdbg", 315 | "request": "launch", 316 | "program": "/opt/vertica/bin/vertica", 317 | "setupCommands": [ 318 | {"text": "directory '${workspaceRoot}/vertica-getdatafromnode/src'"}, 319 | {"text": "target extended-remote 172.17.0.2:2159"}, 320 | {"text": "attach 116"} 321 | ], 322 | "launchCompleteCommand": "None", 323 | "filterStderr": false, 324 | "filterStdout": false, 325 | "externalConsole": false, 326 | "cwd": "${workspaceRoot}", 327 | "logging": { 328 | "engineLogging": true, 329 | "trace": true, 330 | "traceResponse": true 331 | } 332 | } 333 | ] 334 | } 335 | ``` 336 | 337 | ## Advanced host configuration 338 | You'll notice that some checks fail during installation. This is because some checks are indeed checking the host machines due to Docker not being a virtualization layer per se. If you want to have a clean install, consider fixing those below. 339 | 340 | ### Disable CPU frequency scaling 341 | To be performed in the system BIOS 342 | 343 | ### Disable Transparent Huge Pages (THP) 344 | ``` 345 | echo always > /sys/kernel/mm/transparent_hugepage/enabled 346 | ``` 347 | ### Set a proper value to vm.min_free_kbytes 348 | ``` 349 | sysctl vm.min_free_kbytes=$(echo "scale=0;sqrt($(grep MemTotal /proc/meminfo | awk '{printf "%.0f",$2}')*16)" | bc ) 350 | ``` 351 | ### Set a proper value to vm.max_map_count 352 | ``` 353 | sysctl vm.vm.max_map_count=$(echo "$(grep MemTotal /proc/meminfo | awk '{printf "%.0f",$2}')/16" | bc) 354 | ``` 355 | 356 | ### Keep GDB debug info out of the build 357 | With GDB debug infos packages included, the image can get big (1.6GB at the time of this writing). 358 | Excluding them can reduce the image size to about 500MB. 359 | To do that, just add `--build-arg ENABLE_GDB_DEBUG=false` to your build command 360 | --------------------------------------------------------------------------------