├── .github └── workflows │ ├── ci.yml │ └── release.yml ├── .gitignore ├── .gitmodules ├── LICENSE ├── Makefile ├── NEWS.rst ├── README.rst ├── autogen.sh ├── config.mak.in ├── configure.ac ├── etc ├── note.awk ├── pathsep.sed └── quote-lines.sed ├── pgqd.ini ├── src ├── maint.c ├── pgqd.c ├── pgqd.h ├── pgsocket.c ├── pgsocket.h ├── retry.c └── ticker.c └── tests └── test.sh /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # 2 | # https://docs.github.com/en/actions 3 | # https://github.com/actions 4 | # 5 | # mac: https://brew.sh/ 6 | # win: https://www.msys2.org/docs/package-management/ 7 | # win: https://www.archlinux.org/pacman/pacman.8.html 8 | # 9 | 10 | name: CI 11 | 12 | on: 13 | pull_request: {} 14 | push: {} 15 | 16 | jobs: 17 | 18 | test: 19 | name: ${{matrix.test.os}}, pg-${{matrix.test.pgver}} 20 | runs-on: ${{matrix.test.os}} 21 | strategy: 22 | matrix: 23 | test: 24 | - {pgver: "12", os: "ubuntu-latest"} 25 | - {pgver: "14", os: "ubuntu-latest"} 26 | - {pgver: "16", os: "ubuntu-latest"} 27 | - {pgver: "16", os: "macos-latest"} 28 | steps: 29 | - name: "Checkout" 30 | uses: actions/checkout@v4 31 | with: 32 | submodules: true 33 | 34 | - name: "InstallDB / Linux" 35 | if: ${{runner.os == 'Linux'}} 36 | run: | 37 | echo "::group::apt-get-update" 38 | sudo -nH apt-get -q update 39 | sudo -nH apt-get -q install curl ca-certificates gnupg 40 | curl https://www.postgresql.org/media/keys/ACCC4CF8.asc \ 41 | | gpg --dearmor \ 42 | | sudo -nH tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg 43 | echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main ${{matrix.test.pgver}}" \ 44 | | sudo -nH tee /etc/apt/sources.list.d/pgdg.list 45 | sudo -nH apt-get -q update 46 | echo "::endgroup::" 47 | 48 | echo "::group::apt-get-install" 49 | 50 | # disable new cluster creation 51 | sudo -nH mkdir -p /etc/postgresql-common/createcluster.d 52 | echo "create_main_cluster = false" | sudo -nH tee /etc/postgresql-common/createcluster.d/no-main.conf 53 | 54 | sudo -nH apt-get -qyu install \ 55 | postgresql-${{matrix.test.pgver}} \ 56 | postgresql-server-dev-${{matrix.test.pgver}} \ 57 | libevent-dev python3-docutils \ 58 | libpq-dev patchutils 59 | echo "::endgroup::" 60 | 61 | # tune environment 62 | echo "/usr/lib/postgresql/${{matrix.test.pgver}}/bin" >> $GITHUB_PATH 63 | 64 | echo "PGHOST=/tmp" >> $GITHUB_ENV 65 | echo "SED=sed" >> $GITHUB_ENV 66 | echo "RST2MAN=rst2man" >> $GITHUB_ENV 67 | 68 | dpkg -l postgres\* libpq\* gcc\* clang\* libevent\* 69 | 70 | - name: "InstallDB / Mac" 71 | if: ${{runner.os == 'macOS'}} 72 | run: | 73 | echo "::group::install" 74 | brew install patchutils gnu-sed docutils libevent postgresql@${{matrix.test.pgver}} autoconf automake 75 | echo "::endgroup::" 76 | echo "/usr/local/opt/docutils/bin" >> $GITHUB_PATH 77 | echo "/usr/local/opt/postgresql@${{matrix.test.pgver}}/bin" >> $GITHUB_PATH 78 | echo "SED=gsed" >> $GITHUB_ENV 79 | echo "RST2MAN=rst2man.py" >> $GITHUB_ENV 80 | 81 | - name: "Build" 82 | run: | 83 | ./autogen.sh 84 | ./configure --prefix=${GITHUB_WORKSPACE}/testinstall 85 | make RST2MAN=${{env.RST2MAN}} 86 | make install 87 | 88 | - name: "Install PgQ" 89 | run: | 90 | git clone https://github.com/pgq/pgq 91 | make -C pgq 92 | sudo -nH bash -c "PATH='${PATH}' make -C pgq install" 93 | 94 | - name: "StartDB" 95 | run: | 96 | mkdir -p log 97 | LANG=C LC_ALL=C initdb --no-locale data 98 | ${SED} -r -i -e "s,^[# ]*(unix_socket_directories).*,\\1='/tmp'," data/postgresql.conf 99 | pg_ctl -D data -l log/pg.log start || { cat log/pg.log ; exit 1; } 100 | sleep 2 101 | 102 | - name: "Test" 103 | run: make citest 104 | 105 | - name: "StopDB" 106 | run: | 107 | pg_ctl -D data stop 108 | rm -rf data log /tmp/.s.PGSQL* 109 | 110 | mingw: 111 | name: ${{matrix.test.os}}, ${{matrix.test.mingw}} 112 | runs-on: ${{matrix.test.os}} 113 | strategy: 114 | matrix: 115 | test: 116 | #- {os: "windows-latest", arch: i686, mingw: mingw32} 117 | - {os: "windows-latest", arch: x86_64, mingw: mingw64} 118 | steps: 119 | - name: "Checkout" 120 | uses: actions/checkout@v4 121 | with: 122 | submodules: true 123 | 124 | - name: "Setup MSYS" 125 | shell: cmd 126 | run: | 127 | echo C:\msys64\usr\bin>> %GITHUB_PATH% 128 | echo C:\msys64\${{matrix.test.mingw}}\bin>> %GITHUB_PATH% 129 | 130 | - name: "InstallDB / mingw / ${{matrix.test.arch}}" 131 | shell: bash 132 | run: | 133 | # search package lists 134 | pacman -Ss libevent 135 | pacman -Ss autoconf 136 | pacman -Ss automake 137 | pacman -Ss libtool 138 | pacman -Ss postgresql 139 | # install 140 | pacman -S --noconfirm --needed \ 141 | mingw-w64-${{matrix.test.arch}}-libxml2 \ 142 | mingw-w64-${{matrix.test.arch}}-libxslt \ 143 | mingw-w64-${{matrix.test.arch}}-gettext \ 144 | mingw-w64-${{matrix.test.arch}}-postgresql \ 145 | mingw-w64-${{matrix.test.arch}}-python-docutils \ 146 | mingw-w64-${{matrix.test.arch}}-libevent \ 147 | mingw-w64-${{matrix.test.arch}}-pkgconf \ 148 | autoconf automake libtool pkgconf 149 | INCDIR=$(pg_config --includedir) 150 | PG_CPPFLAGS="-I${INCDIR}" 151 | echo "PG_CPPFLAGS=${PG_CPPFLAGS}" >> $GITHUB_ENV 152 | echo "PG_CPPFLAGS=${PG_CPPFLAGS}" 153 | echo "PATH=$PATH" 154 | 155 | - name: "Build" 156 | shell: bash 157 | run: | 158 | workspace=$(echo "${GITHUB_WORKSPACE}" | sed -f etc/pathsep.sed) 159 | ./autogen.sh 160 | ./configure --prefix="${workspace}/testinstall" 161 | make 162 | make install 163 | ./pgqd -V 164 | 165 | - name: "Install PgQ" 166 | shell: bash 167 | run: | 168 | git clone https://github.com/pgq/pgq 169 | make -C pgq 170 | make -C pgq install 171 | 172 | - name: "StartDB" 173 | shell: bash 174 | run: | 175 | mkdir log 176 | initdb.exe --no-locale -U postgres -D data 177 | pg_ctl -D data -l log/pg.log start || { cat log/pg.log ; exit 1; } 178 | sleep 3 179 | 180 | - name: "Test" 181 | if: false 182 | shell: bash 183 | run: make citest 184 | 185 | - name: "StopDB" 186 | shell: bash 187 | run: | 188 | pg_ctl -D data stop 189 | 190 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | 2 | name: REL 3 | 4 | on: 5 | push: 6 | tags: ["v[0-9]*"] 7 | 8 | jobs: 9 | release: 10 | name: Release 11 | runs-on: ubuntu-latest 12 | steps: 13 | 14 | - name: Checkout code 15 | id: checkout 16 | uses: actions/checkout@v4 17 | with: 18 | submodules: true 19 | 20 | - name: Build tarball 21 | id: build 22 | run: | 23 | sudo -nH apt-get -u -y install pandoc autoconf automake libtool \ 24 | libevent-dev libpq-dev patchutils python3-docutils 25 | ./autogen.sh 26 | ./configure 27 | make checkver 28 | make dist 29 | 30 | PACKAGE=$(grep ^PACKAGE_NAME config.mak | sed 's/.*= *//') 31 | VERSION=$(grep ^PACKAGE_VERSION config.mak | sed 's/.*= *//') 32 | # default - gh:release 33 | # PRERELEASE - gh:prerelease 34 | # DRAFT - gh:draft,prerelease 35 | PRERELEASE="false" 36 | DRAFT="false" 37 | if echo "${VERSION}" | grep -qE '(a|b|rc)'; then PRERELEASE="true"; fi 38 | if echo "${VERSION}" | grep -qE '(dev)'; then DRAFT="true"; PRERELEASE="true"; fi 39 | 40 | test "${{github.ref}}" = "refs/tags/v${VERSION}" || { echo "ERR: tag mismatch"; exit 1; } 41 | echo "PACKAGE=${PACKAGE}" >> $GITHUB_ENV 42 | echo "VERSION=${VERSION}" >> $GITHUB_ENV 43 | echo "TGZ=${PACKAGE}-${VERSION}.tar.gz" >> $GITHUB_ENV 44 | echo "PRERELEASE=${PRERELEASE}" >> $GITHUB_ENV 45 | echo "DRAFT=${DRAFT}" >> $GITHUB_ENV 46 | 47 | pandoc --version 48 | mkdir -p tmp 49 | make -s shownote > tmp/note.md 50 | cat tmp/note.md 51 | 52 | - name: "Create Github release" 53 | env: 54 | GH_TOKEN: ${{secrets.GITHUB_TOKEN}} 55 | run: | 56 | title="${PACKAGE} v${VERSION}" 57 | ghf="--notes-file=tmp/note.md" 58 | if test "${DRAFT}" = "true"; then ghf="${ghf} --draft"; fi 59 | if test "${PRERELEASE}" = "true"; then ghf="${ghf} --prerelease"; fi 60 | gh release create "v${VERSION}" "${TGZ}" --title="${title}" ${ghf} 61 | 62 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .objs 2 | install-sh 3 | pgqd 4 | pgqd.ini.h 5 | configure 6 | config.mak 7 | config.log 8 | config.guess 9 | config.status 10 | config.sub 11 | pgqd.1 12 | 13 | *.swp 14 | tmp 15 | 16 | *.debhelper 17 | *.log 18 | *.substvars 19 | *-stamp 20 | debian/files 21 | tests/log 22 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "lib"] 2 | path = lib 3 | url = https://github.com/libusual/libusual 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009 Marko Kreen 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | -include config.mak 3 | 4 | PG_CONFIG ?= pg_config 5 | PG_INCDIR = $(shell $(PG_CONFIG) --includedir) 6 | PG_LIBDIR = $(shell $(PG_CONFIG) --libdir) 7 | RST2MAN = rst2man 8 | 9 | bin_PROGRAMS = pgqd 10 | man_MANS = pgqd.1 11 | 12 | pgqd_SOURCES = src/pgqd.c src/maint.c src/ticker.c src/retry.c \ 13 | src/pgsocket.c src/pgsocket.h \ 14 | src/pgqd.h 15 | nodist_pgqd_SOURCES = pgqd.ini.h 16 | pgqd_CPPFLAGS = -I$(PG_INCDIR) -Isrc -I. 17 | pgqd_LDFLAGS = -L$(PG_LIBDIR) 18 | pgqd_LIBS = -lpq -lm 19 | 20 | pgqd_EMBED_LIBUSUAL = 1 21 | USUAL_DIR = lib 22 | AM_FEATURES = libusual 23 | 24 | EXTRA_DIST = pgqd.ini autogen.sh configure.ac Makefile \ 25 | README.rst NEWS.rst tests/test.sh \ 26 | lib/find_modules.sh \ 27 | lib/mk/antimake.mk lib/mk/amext-libusual.mk \ 28 | lib/mk/install-sh lib/mk/std-autogen.sh \ 29 | config.mak.in lib/usual/config.h.in \ 30 | lib/m4/antimake.m4 \ 31 | lib/m4/ax_pthread.m4 \ 32 | lib/m4/usual.m4 \ 33 | configure config.sub config.guess install-sh 34 | CLEANFILES = pgqd.ini.h 35 | 36 | CONFIG_H = $(USUAL_DIR)/lib/usual/config.h 37 | 38 | include $(USUAL_DIR)/mk/antimake.mk 39 | 40 | pgqd.ini.h: pgqd.ini 41 | sed -f etc/quote-lines.sed $< > $@ 42 | 43 | install: install-conf 44 | install-conf: 45 | mkdir -p '$(DESTDIR)$(docdir)/conf' 46 | $(INSTALL) -m 644 pgqd.ini '$(DESTDIR)$(docdir)/conf/pgqd.ini.templ' 47 | 48 | tags: 49 | ctags src/*.[ch] lib/usual/*.[ch] 50 | 51 | configure: 52 | ./autogen.sh 53 | 54 | #config.mak: configure 55 | # ./configure 56 | 57 | *.o: $(CONFIG_H) 58 | 59 | $(CONFIG_H): 60 | $(error Please run ./configure first) 61 | 62 | xclean: clean 63 | rm -f config.mak config.guess config.sub config.log config.sub config.status 64 | rm -f configure install-sh lib/usual/config.h 65 | 66 | pgqd.1: README.rst 67 | $(RST2MAN) $< > $@ 68 | 69 | citest: check 70 | 71 | check: 72 | ./tests/test.sh 73 | 74 | # PACKAGE_VERSION 75 | VERSION = $(shell ./configure --version | head -n 1 | sed -e 's/.* //') 76 | RXVERSION = $(shell echo $(VERSION) | sed 's/\./[.]/g') 77 | NEWS = NEWS.rst 78 | TAG = v$(VERSION) 79 | 80 | checkver: 81 | @echo "Checking version" 82 | @test -f configure || { echo "need ./configure"; exit 1; } 83 | @grep -q '^pgqd $(RXVERSION)\b' $(NEWS) \ 84 | || { echo "Version '$(VERSION)' not in $(NEWS)"; exit 1; } 85 | @echo "Checking git repo" 86 | @git diff --stat --exit-code || { echo "ERROR: Unclean repo"; exit 1; } 87 | 88 | release: checkver 89 | git tag $(TAG) 90 | git push github $(TAG):$(TAG) 91 | 92 | unrelease: 93 | git push github :$(TAG) 94 | git tag -d $(TAG) 95 | 96 | shownote: 97 | awk -v VER="$(VERSION)" -f etc/note.awk $(NEWS) \ 98 | | pandoc -f rst -t gfm --wrap=none 99 | 100 | -------------------------------------------------------------------------------- /NEWS.rst: -------------------------------------------------------------------------------- 1 | NEWS 2 | ==== 3 | 4 | pgqd 3.5 5 | -------- 6 | 7 | * Quote dbname in logs. 8 | * Upgrade libusual. 9 | * Add CI workflows. 10 | 11 | pgqd 3.4 12 | -------- 13 | 14 | * Upgrade libusual. 15 | * Switch to direct libevent2 usage. 16 | * Fix ``make dist``. 17 | * Add manpage. 18 | 19 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | 2 | ==== 3 | pgqd 4 | ==== 5 | 6 | -------------------------- 7 | Maintenance daemon for PgQ 8 | -------------------------- 9 | 10 | :Manual section: 1 11 | 12 | Synopsis 13 | ======== 14 | 15 | pgqd [-qvd] config 16 | 17 | pgqd [-skr] config 18 | 19 | pgqd --ini|-h|-V 20 | 21 | Description 22 | =========== 23 | 24 | Runs both ticker and periodic maintenence for all 25 | databases in one PostgreSQL cluster. 26 | 27 | Options 28 | ======= 29 | 30 | -q Do not log to stdout 31 | -v Verbose log 32 | -d Daemonize process 33 | -s Send SIGINT to running process to stop it 34 | -k Send SIGTERM to running process to stop it 35 | -r Send SIGHUP to running process to reload config 36 | -h Show help 37 | -V Show version 38 | --ini Show sample config 39 | 40 | Configuration 41 | ============= 42 | 43 | Config uses `ini` file syntax:: 44 | 45 | [pgqd] 46 | logfile = ~/log/pgqd.log 47 | pidfile = ~/pid/pgqd.pid 48 | 49 | Options: 50 | 51 | logfile 52 | Filename to log to. 53 | Default: empty. 54 | 55 | pidfile 56 | Filename to store pid, required when daemonizing. 57 | Default: empty. 58 | 59 | base_connstr 60 | Connect string without dbname= 61 | Default: empty. 62 | 63 | initial_database 64 | Startup db to query other databases. 65 | Default: template1 66 | 67 | database_list 68 | Limit ticker to specific databases. 69 | Default: empty, which means all database. 70 | 71 | syslog 72 | Whether to log into syslog. 73 | Default: 1 74 | 75 | syslog_ident 76 | Name to use for syslog. 77 | Default: pgqd 78 | 79 | check_period 80 | How often to check for new databases, in seconds. 81 | Default: 60. 82 | 83 | retry_period 84 | How often to flush retry queue, in seconds. 85 | Default: 30 86 | 87 | maint_period 88 | How often to do maintentance, in seconds. 89 | Default: 120 90 | 91 | ticker_period 92 | How often to run ticker, in seconds. 93 | Default: 1 94 | 95 | 96 | Install 97 | ======= 98 | 99 | pgqd uses autoconf based build system:: 100 | 101 | ./configure --prefix=/opt 102 | make 103 | make install 104 | 105 | Dependencies: libevent, python3-docutils 106 | 107 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | ./lib/mk/std-autogen.sh lib 4 | 5 | -------------------------------------------------------------------------------- /config.mak.in: -------------------------------------------------------------------------------- 1 | 2 | PACKAGE_NAME = @PACKAGE_NAME@ 3 | PACKAGE_TARNAME = @PACKAGE_TARNAME@ 4 | PACKAGE_VERSION = @PACKAGE_VERSION@ 5 | PACKAGE_STRING = @PACKAGE_STRING@ 6 | 7 | SUFFIX = 8 | 9 | prefix = @prefix@ 10 | datarootdir = @datarootdir@ 11 | exec_prefix = @exec_prefix@ 12 | datadir = @datadir@ 13 | docdir = @docdir@$(SUFFIX) 14 | mandir = @mandir@ 15 | bindir = @bindir@ 16 | 17 | PG_CONFIG = @PG_CONFIG@ 18 | 19 | DESTDIR = 20 | 21 | SED = @SED@ 22 | GREP = @GREP@ 23 | EGREP = @EGREP@ 24 | MKDIR_P = @MKDIR_P@ 25 | LN_S = @LN_S@ 26 | 27 | CC = @CC@ 28 | CPPFLAGS = @CPPFLAGS@ 29 | CFLAGS = @CFLAGS@ @WFLAGS@ 30 | LDFLAGS = @LDFLAGS@ 31 | LIBS = @LIBS@ 32 | 33 | SHELL = @SHELL@ 34 | INSTALL = @INSTALL@ 35 | INSTALL_PROGRAM = @INSTALL_PROGRAM@ 36 | INSTALL_SCRIPT = @INSTALL_SCRIPT@ 37 | INSTALL_DATA = @INSTALL_DATA@ 38 | BININSTALL = $(INSTALL_SCRIPT) 39 | 40 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | dnl Process this file with autoconf to produce a configure script. 2 | 3 | AC_INIT(pgqd, 3.5) 4 | AC_CONFIG_SRCDIR(src/pgqd.c) 5 | AC_CONFIG_HEADER(lib/usual/config.h) 6 | AC_PREREQ([2.59]) 7 | 8 | dnl Find PostgreSQL pg_config 9 | AC_ARG_WITH(pgconfig, [ --with-pgconfig=PG_CONFIG path to pg_config (default: pg_config)], 10 | [ AC_MSG_CHECKING(for pg_config) 11 | PG_CONFIG=$withval 12 | AC_MSG_RESULT($PG_CONFIG)], 13 | [ AC_PATH_PROGS(PG_CONFIG, pg_config) ]) 14 | test -n "$PG_CONFIG" || AC_MSG_ERROR([Cannot continue without pg_config]) 15 | 16 | AC_USUAL_PORT_CHECK 17 | AC_USUAL_PROGRAM_CHECK 18 | AC_USUAL_HEADER_CHECK 19 | AC_USUAL_TYPE_CHECK 20 | AC_USUAL_FUNCTION_CHECK 21 | 22 | dnl Postres headers on Solaris define incompat unsetenv without that 23 | AC_CHECK_FUNCS(unsetenv) 24 | 25 | dnl Require libevent 26 | PKG_PROG_PKG_CONFIG 27 | PKG_CHECK_MODULES(LIBEVENT, [libevent >= 2.0], [ 28 | CFLAGS="${CFLAGS} ${LIBEVENT_CFLAGS}" 29 | LIBS="${LIBS} ${LIBEVENT_LIBS}" 30 | ], [AC_MSG_ERROR([pgqd requires libevent >= 2.0])]) 31 | 32 | dnl Needed on SmartOS (Solaris) 33 | AC_SEARCH_LIBS([socket],[socket]) 34 | 35 | AC_USUAL_DEBUG 36 | AC_USUAL_CASSERT 37 | 38 | dnl Write result 39 | AC_CONFIG_FILES([config.mak]) 40 | AC_OUTPUT 41 | -------------------------------------------------------------------------------- /etc/note.awk: -------------------------------------------------------------------------------- 1 | # extract version notes for version VER 2 | 3 | /^[-_0-9a-zA-Z]+ v?[0-9]/ { 4 | if ($2 == VER) { 5 | good = 1 6 | next 7 | } else { 8 | good = 0 9 | } 10 | } 11 | 12 | /^(===|---)/ { next } 13 | 14 | { 15 | if (good) { 16 | # also remove sphinx syntax 17 | print gensub(/:(\w+):`~?([^`]+)`/, "``\\2``", "g") 18 | } 19 | } 20 | 21 | -------------------------------------------------------------------------------- /etc/pathsep.sed: -------------------------------------------------------------------------------- 1 | s,\\,/,g 2 | -------------------------------------------------------------------------------- /etc/quote-lines.sed: -------------------------------------------------------------------------------- 1 | s/.*/"&\\n"/ 2 | -------------------------------------------------------------------------------- /pgqd.ini: -------------------------------------------------------------------------------- 1 | [pgqd] 2 | 3 | # where to log 4 | logfile = ~/log/pgqd.log 5 | 6 | # pidfile 7 | pidfile = ~/pid/pgqd.pid 8 | 9 | ## optional parameters ## 10 | 11 | # libpq connect string without dbname= 12 | #base_connstr = 13 | 14 | # startup db to query other databases 15 | #initial_database = template1 16 | 17 | # limit ticker to specific databases 18 | #database_list = 19 | 20 | # log into syslog 21 | #syslog = 1 22 | #syslog_ident = pgqd 23 | 24 | ## optional timeouts ## 25 | 26 | # how often to check for new databases 27 | #check_period = 60 28 | 29 | # how often to flush retry queue 30 | #retry_period = 30 31 | 32 | # how often to do maintentance 33 | #maint_period = 120 34 | 35 | # how often to run ticker 36 | #ticker_period = 1 37 | 38 | -------------------------------------------------------------------------------- /src/maint.c: -------------------------------------------------------------------------------- 1 | #include "pgqd.h" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | struct MaintOp { 9 | struct List head; 10 | const char *func_name; 11 | const char *func_arg; 12 | }; 13 | 14 | static struct MaintOp *next_op(struct PgDatabase *db) 15 | { 16 | struct List *el = statlist_pop(&db->maint_op_list); 17 | if (!el) 18 | return NULL; 19 | return container_of(el, struct MaintOp, head); 20 | } 21 | 22 | static void free_op(struct MaintOp *op) 23 | { 24 | if (op) { 25 | free((void *)op->func_name); 26 | free((void *)op->func_arg); 27 | free(op); 28 | } 29 | } 30 | 31 | void free_maint(struct PgDatabase *db) 32 | { 33 | struct MaintOp *op; 34 | 35 | strlist_free(db->maint_item_list); 36 | db->maint_item_list = NULL; 37 | 38 | while ((op = next_op(db)) != NULL) { 39 | free_op(op); 40 | } 41 | free_op(db->cur_maint); 42 | db->cur_maint = NULL; 43 | } 44 | 45 | static void close_maint(struct PgDatabase *db, double sleep_time) 46 | { 47 | log_debug("%s: close_maint, %f", db->logname, sleep_time); 48 | db->maint_state = DB_CLOSED; 49 | pgs_reconnect(db->c_maint, sleep_time); 50 | } 51 | 52 | static void run_test_version(struct PgDatabase *db) 53 | { 54 | const char *q = "select 1 from pg_proc p, pg_namespace n" 55 | " where p.pronamespace = n.oid" 56 | " and p.proname = 'maint_operations'" 57 | " and n.nspname = 'pgq'"; 58 | log_debug("%s: %s", db->logname, q); 59 | pgs_send_query_simple(db->c_maint, q); 60 | db->maint_state = DB_MAINT_TEST_VERSION; 61 | } 62 | 63 | static bool has_ops(PGresult *res) 64 | { 65 | if (PQntuples(res) == 1 && atoi(PQgetvalue(res, 0, 0)) == 1) 66 | return true; 67 | return false; 68 | } 69 | 70 | static bool fill_op_list(struct PgDatabase *db, PGresult *res) 71 | { 72 | int i; 73 | struct MaintOp *op = NULL; 74 | const char *fname, *farg; 75 | 76 | free_maint(db); 77 | 78 | for (i = 0; i < PQntuples(res); i++) { 79 | op = calloc(1, sizeof(*op)); 80 | if (!op) 81 | return false; 82 | list_init(&op->head); 83 | fname = PQgetvalue(res, i, 0); 84 | farg = NULL; 85 | if (!PQgetisnull(res, i, 1)) 86 | farg = PQgetvalue(res, i, 1); 87 | log_debug("load_op: %s / %s", fname, farg ? farg : "NULL"); 88 | op->func_name = strdup(fname); 89 | if (!op->func_name) 90 | goto failed; 91 | if (farg) { 92 | op->func_arg = strdup(farg); 93 | if (!op->func_arg) 94 | goto failed; 95 | } 96 | statlist_append(&db->maint_op_list, &op->head); 97 | } 98 | return true; 99 | failed: 100 | free_op(op); 101 | return false; 102 | } 103 | 104 | static void run_op_list(struct PgDatabase *db) 105 | { 106 | const char *q = "select func_name, func_arg from pgq.maint_operations()"; 107 | log_debug("%s: %s", db->logname, q); 108 | pgs_send_query_simple(db->c_maint, q); 109 | db->maint_state = DB_MAINT_LOAD_OPS; 110 | } 111 | 112 | static const char *stmt_names[] = { 113 | "vacuum", 114 | "vacuum analyze", 115 | NULL 116 | }; 117 | 118 | static void run_op(struct PgDatabase *db, PGresult *res) 119 | { 120 | struct MaintOp *op; 121 | char buf[1024]; 122 | char namebuf[256]; 123 | const char **np; 124 | 125 | if (db->cur_maint) { 126 | if (res && PQntuples(res) > 0) { 127 | const char *val = PQgetvalue(res, 0, 0); 128 | if (val && atoi(val)) { 129 | op = db->cur_maint; 130 | goto repeat; 131 | } 132 | } 133 | next: 134 | free_op(db->cur_maint); 135 | db->cur_maint = NULL; 136 | } 137 | op = next_op(db); 138 | if (!op) { 139 | stats.n_maint++; 140 | close_maint(db, cf.maint_period); 141 | return; 142 | } 143 | db->cur_maint = op; 144 | repeat: 145 | /* check if its magic statement */ 146 | for (np = stmt_names; *np; np++) { 147 | if (strcasecmp(op->func_name, *np) != 0) 148 | continue; 149 | if (!pg_quote_fqident(namebuf, op->func_arg, sizeof(namebuf))) { 150 | log_error("Bad table name? - %s", op->func_arg); 151 | goto next; 152 | } 153 | /* run as a statement */ 154 | snprintf(buf, sizeof(buf), "%s %s", op->func_name, namebuf); 155 | log_debug("%s: [%s]", db->logname, buf); 156 | pgs_send_query_simple(db->c_maint, buf); 157 | goto done; 158 | } 159 | 160 | /* run as a function */ 161 | if (!pg_quote_fqident(namebuf, op->func_name, sizeof(namebuf))) { 162 | log_error("Bad func name? - %s", op->func_name); 163 | goto next; 164 | } 165 | if (op->func_arg) { 166 | snprintf(buf, sizeof(buf), "select %s($1)", namebuf); 167 | log_debug("%s: [%s]", db->logname, buf); 168 | pgs_send_query_params(db->c_maint, buf, 1, op->func_arg); 169 | } else { 170 | snprintf(buf, sizeof(buf), "select %s()", namebuf); 171 | log_debug("%s: [%s]", db->logname, buf); 172 | pgs_send_query_simple(db->c_maint, buf); 173 | } 174 | done: 175 | db->maint_state = DB_MAINT_OP; 176 | } 177 | 178 | static bool fill_items(struct PgDatabase *db, PGresult *res) 179 | { 180 | int i; 181 | if (db->maint_item_list) 182 | strlist_free(db->maint_item_list); 183 | db->maint_item_list = strlist_new(USUAL_ALLOC); 184 | if (!db->maint_item_list) 185 | return false; 186 | for (i = 0; i < PQntuples(res); i++) { 187 | const char *item = PQgetvalue(res, i, 0); 188 | if (item) 189 | if (!strlist_append(db->maint_item_list, item)) 190 | return false; 191 | } 192 | return true; 193 | } 194 | 195 | static void run_queue_list(struct PgDatabase *db) 196 | { 197 | const char *q = "select queue_name from pgq.get_queue_info()"; 198 | log_debug("%s: %s", db->logname, q); 199 | pgs_send_query_simple(db->c_maint, q); 200 | db->maint_state = DB_MAINT_LOAD_QUEUES; 201 | } 202 | 203 | static void run_vacuum_list(struct PgDatabase *db) 204 | { 205 | const char *q = "select * from pgq.maint_tables_to_vacuum()"; 206 | log_debug("%s: %s", db->logname, q); 207 | pgs_send_query_simple(db->c_maint, q); 208 | db->maint_state = DB_MAINT_VACUUM_LIST; 209 | } 210 | 211 | static void run_rotate_part1(struct PgDatabase *db) 212 | { 213 | const char *q; 214 | const char *qname; 215 | qname = strlist_pop(db->maint_item_list); 216 | q = "select pgq.maint_rotate_tables_step1($1)"; 217 | log_debug("%s: %s [%s]", db->logname, q, qname); 218 | pgs_send_query_params(db->c_maint, q, 1, qname); 219 | free((void *)qname); 220 | db->maint_state = DB_MAINT_ROT1; 221 | } 222 | 223 | static void run_rotate_part2(struct PgDatabase *db) 224 | { 225 | const char *q = "select pgq.maint_rotate_tables_step2()"; 226 | log_debug("%s: %s", db->logname, q); 227 | pgs_send_query_simple(db->c_maint, q); 228 | db->maint_state = DB_MAINT_ROT2; 229 | } 230 | 231 | static void run_vacuum(struct PgDatabase *db) 232 | { 233 | char qbuf[256]; 234 | const char *table; 235 | table = strlist_pop(db->maint_item_list); 236 | snprintf(qbuf, sizeof(qbuf), "vacuum %s", table); 237 | log_debug("%s: %s", db->logname, qbuf); 238 | pgs_send_query_simple(db->c_maint, qbuf); 239 | free((void *)table); 240 | db->maint_state = DB_MAINT_DO_VACUUM; 241 | } 242 | 243 | static void maint_handler(struct PgSocket *s, void *arg, enum PgEvent ev, PGresult *res) 244 | { 245 | struct PgDatabase *db = arg; 246 | 247 | switch (ev) { 248 | case PGS_CONNECT_OK: 249 | log_debug("%s: starting maintenance", db->logname); 250 | if (db->has_maint_operations) 251 | run_op_list(db); 252 | else 253 | run_test_version(db); 254 | break; 255 | case PGS_RESULT_OK: 256 | if (PQresultStatus(res) != PGRES_TUPLES_OK) { 257 | close_maint(db, 20); 258 | return; 259 | } 260 | switch (db->maint_state) { 261 | case DB_MAINT_TEST_VERSION: 262 | if (has_ops(res)) { 263 | db->has_maint_operations = true; 264 | run_op_list(db); 265 | } else { 266 | run_queue_list(db); 267 | } 268 | break; 269 | case DB_MAINT_LOAD_OPS: 270 | if (!fill_op_list(db, res)) 271 | goto mem_err; 272 | /* fallthrough */ 273 | case DB_MAINT_OP: 274 | run_op(db, res); 275 | break; 276 | case DB_MAINT_LOAD_QUEUES: 277 | if (!fill_items(db, res)) 278 | goto mem_err; 279 | /* fallthrough */ 280 | case DB_MAINT_ROT1: 281 | if (!strlist_empty(db->maint_item_list)) { 282 | run_rotate_part1(db); 283 | } else { 284 | run_rotate_part2(db); 285 | } 286 | break; 287 | case DB_MAINT_ROT2: 288 | run_vacuum_list(db); 289 | break; 290 | case DB_MAINT_VACUUM_LIST: 291 | if (!fill_items(db, res)) 292 | goto mem_err; 293 | /* fallthrough */ 294 | case DB_MAINT_DO_VACUUM: 295 | if (!strlist_empty(db->maint_item_list)) { 296 | run_vacuum(db); 297 | } else { 298 | close_maint(db, cf.maint_period); 299 | } 300 | break; 301 | default: 302 | fatal("bad state"); 303 | break; 304 | } 305 | break; 306 | case PGS_TIMEOUT: 307 | log_debug("%s: maint timeout", db->logname); 308 | if (!pgs_connection_valid(db->c_maint)) 309 | launch_maint(db); 310 | else 311 | run_queue_list(db); 312 | break; 313 | default: 314 | log_warning("%s: default reconnect", db->logname); 315 | pgs_reconnect(db->c_maint, 60); 316 | break; 317 | } 318 | return; 319 | mem_err: 320 | if (db->maint_item_list) { 321 | strlist_free(db->maint_item_list); 322 | db->maint_item_list = NULL; 323 | } 324 | pgs_disconnect(db->c_maint); 325 | pgs_sleep(db->c_maint, 20); 326 | } 327 | 328 | void launch_maint(struct PgDatabase *db) 329 | { 330 | char *cstr; 331 | 332 | log_debug("%s: launch_maint", db->logname); 333 | 334 | if (!db->c_maint) { 335 | if (db->maint_item_list) { 336 | strlist_free(db->maint_item_list); 337 | db->maint_item_list = NULL; 338 | } 339 | cstr = make_connstr(db->name); 340 | if (!cstr) { 341 | log_error("make_connstr: %s", strerror(errno)); 342 | return; 343 | } 344 | db->c_maint = pgs_create(cstr, maint_handler, db, ev_base); 345 | free(cstr); 346 | if (!db->c_maint) { 347 | log_error("pgs_create: %s", strerror(errno)); 348 | return; 349 | } 350 | } 351 | 352 | if (!pgs_connection_valid(db->c_maint)) { 353 | pgs_connect(db->c_maint); 354 | } else { 355 | /* Already have a connection, what are we doing here */ 356 | log_error("%s: maint already initialized", db->logname); 357 | return; 358 | } 359 | } 360 | 361 | -------------------------------------------------------------------------------- /src/pgqd.c: -------------------------------------------------------------------------------- 1 | #include "pgqd.h" 2 | 3 | #include 4 | 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | static void detect_dbs(void); 14 | static void recheck_dbs(void); 15 | 16 | static const char usage_str[] = 17 | "usage: pgq-ticker [switches] config.file\n" 18 | "Switches:\n" 19 | " -v Increase verbosity\n" 20 | " -q No output to console\n" 21 | " -d Daemonize\n" 22 | " -h Show help\n" 23 | " -V Show version\n" 24 | " --ini Show sample config file\n" 25 | " -s Stop - send SIGINT to running process\n" 26 | " -k Kill - send SIGTERM to running process\n" 27 | #ifdef SIGHUP 28 | " -r Reload - send SIGHUP to running process\n" 29 | #endif 30 | ""; 31 | 32 | static const char *sample_ini = 33 | #include "pgqd.ini.h" 34 | ; 35 | 36 | struct Config cf; 37 | 38 | struct Stats stats; 39 | 40 | static struct PgSocket *db_template; 41 | 42 | static STATLIST(database_list); 43 | 44 | static int got_sigint; 45 | 46 | static struct event *ev_stats; 47 | static struct event *ev_sigterm; 48 | static struct event *ev_sigint; 49 | #ifdef SIGHUP 50 | static struct event *ev_sighup; 51 | #endif 52 | 53 | struct event_base *ev_base; 54 | 55 | #define CF_REL_BASE struct Config 56 | static const struct CfKey conf_params[] = { 57 | CF_ABS("logfile", CF_FILE, cf_logfile, 0, NULL), 58 | CF_REL("pidfile", CF_FILE, pidfile, 0, NULL), 59 | CF_REL("initial_database", CF_STR, initial_database, 0, "template1"), 60 | CF_REL("base_connstr", CF_STR, base_connstr, 0, ""), 61 | CF_REL("database_list", CF_STR, database_list, 0, NULL), 62 | CF_ABS("syslog", CF_INT, cf_syslog, 0, "1"), 63 | CF_ABS("syslog_ident", CF_STR, cf_syslog_ident, 0, "pgqd"), 64 | CF_ABS("syslog_facility", CF_STR, cf_syslog_facility, 0, "daemon"), 65 | CF_REL("check_period", CF_TIME_DOUBLE, check_period, 0, "60"), 66 | CF_REL("maint_period", CF_TIME_DOUBLE, maint_period, 0, "120"), 67 | CF_REL("retry_period", CF_TIME_DOUBLE, retry_period, 0, "30"), 68 | CF_REL("ticker_period", CF_TIME_DOUBLE, ticker_period, 0, "1"), 69 | CF_REL("stats_period", CF_TIME_DOUBLE, stats_period, 0, "30"), 70 | CF_REL("connection_lifetime", CF_TIME_DOUBLE, connection_lifetime, 0, "3600"), 71 | { NULL }, 72 | }; 73 | 74 | static const struct CfSect conf_sects[] = { 75 | { "pgqd", conf_params }, 76 | { NULL } 77 | }; 78 | 79 | static struct CfContext conf_info = { 80 | .sect_list = conf_sects, 81 | .base = &cf, 82 | }; 83 | 84 | static void load_config(void) 85 | { 86 | bool ok = cf_load_file(&conf_info, cf.config_file); 87 | if (!ok) 88 | fatal("failed to read config"); 89 | reset_logging(); 90 | } 91 | 92 | static void handle_sigterm(evutil_socket_t sock, short flags, void *arg) 93 | { 94 | log_info("Got SIGTERM, fast exit"); 95 | /* pidfile cleanup happens via atexit() */ 96 | exit(1); 97 | } 98 | 99 | static void handle_sigint(evutil_socket_t sock, short flags, void *arg) 100 | { 101 | log_info("Got SIGINT, shutting down"); 102 | /* notify main loop to exit */ 103 | got_sigint = 1; 104 | } 105 | 106 | #ifdef SIGHUP 107 | static void handle_sighup(evutil_socket_t sock, short flags, void *arg) 108 | { 109 | log_info("Got SIGHUP, re-reading config"); 110 | load_config(); 111 | recheck_dbs(); 112 | } 113 | #endif 114 | 115 | static void signal_setup(void) 116 | { 117 | int err; 118 | 119 | #ifdef SIGPIPE 120 | sigset_t set; 121 | 122 | /* block SIGPIPE */ 123 | sigemptyset(&set); 124 | sigaddset(&set, SIGPIPE); 125 | err = sigprocmask(SIG_BLOCK, &set, NULL); 126 | if (err < 0) 127 | fatal_perror("sigprocmask"); 128 | #endif 129 | 130 | #ifdef SIGHUP 131 | /* catch signals */ 132 | ev_sighup = evsignal_new(ev_base, SIGHUP, handle_sighup, NULL); 133 | if (!ev_sighup) 134 | fatal_perror("evsignal_new"); 135 | err = evsignal_add(ev_sighup, NULL); 136 | if (err < 0) 137 | fatal_perror("evsignal_add"); 138 | #endif 139 | 140 | ev_sigterm = evsignal_new(ev_base, SIGTERM, handle_sigterm, NULL); 141 | if (!ev_sigterm) 142 | fatal_perror("evsignal_new"); 143 | err = evsignal_add(ev_sigterm, NULL); 144 | if (err < 0) 145 | fatal_perror("evsignal_add"); 146 | 147 | ev_sigint = evsignal_new(ev_base, SIGINT, handle_sigint, NULL); 148 | if (!ev_sigint) 149 | fatal_perror("evsignal_new"); 150 | err = evsignal_add(ev_sigint, NULL); 151 | if (err < 0) 152 | fatal_perror("signal_add"); 153 | } 154 | 155 | char *make_connstr(const char *dbname) 156 | { 157 | size_t buflen; 158 | char *buf, *dst; 159 | const char *src; 160 | 161 | buflen = strlen(cf.base_connstr) + strlen(dbname) * 2 + 32; 162 | buf = calloc(1, buflen); 163 | if (!buf) 164 | return NULL; 165 | snprintf(buf, buflen, "%s dbname='", cf.base_connstr); 166 | dst = buf + strlen(buf); 167 | for (src = dbname; *src; src++) { 168 | if (*src == '\'' || *src == '\\') { 169 | *dst++ = '\\'; 170 | } 171 | *dst++ = *src; 172 | } 173 | *dst = '\''; 174 | return buf; 175 | } 176 | 177 | static char *safe_dbname(const char *dbname) 178 | { 179 | char *buf, *dst; 180 | const char *src; 181 | size_t buflen; 182 | 183 | buflen = strlen(dbname) * 2 + 3; 184 | buf = calloc(1, buflen); 185 | if (!buf) 186 | return NULL; 187 | dst = buf; 188 | *dst++ = '['; 189 | for (src = dbname; *src; src++) { 190 | if ((unsigned char)*src < ' ') { 191 | *dst++ = '?'; 192 | } else { 193 | *dst++ = *src; 194 | } 195 | } 196 | *dst++ = ']'; 197 | *dst++ = '\0'; 198 | return buf; 199 | } 200 | 201 | static void launch_db(const char *dbname) 202 | { 203 | struct PgDatabase *db; 204 | struct List *elem; 205 | 206 | /* check of already exists */ 207 | statlist_for_each(elem, &database_list) { 208 | db = container_of(elem, struct PgDatabase, head); 209 | if (strcmp(db->name, dbname) == 0) { 210 | db->dropped = false; 211 | return; 212 | } 213 | } 214 | 215 | /* create new db entry */ 216 | db = calloc(1, sizeof(*db)); 217 | if (!db) { 218 | log_error("calloc: %s", strerror(errno)); 219 | return; 220 | } 221 | db->name = strdup(dbname); 222 | if (!db->name) { 223 | log_error("strdup: %s", strerror(errno)); 224 | free(db); 225 | return; 226 | } 227 | db->logname = safe_dbname(dbname); 228 | if (!db->logname) { 229 | log_error("safe_dbname: %s", strerror(errno)); 230 | free((void *)db->name); 231 | free(db); 232 | return; 233 | } 234 | list_init(&db->head); 235 | statlist_init(&db->maint_op_list, "maint_op_list"); 236 | statlist_append(&database_list, &db->head); 237 | 238 | /* start working on it */ 239 | launch_ticker(db); 240 | } 241 | 242 | static void drop_db(struct PgDatabase *db, bool log) 243 | { 244 | if (log) 245 | log_info("Unregister database: %s", db->logname); 246 | statlist_remove(&database_list, &db->head); 247 | pgs_free(db->c_ticker); 248 | pgs_free(db->c_maint); 249 | pgs_free(db->c_retry); 250 | free_maint(db); 251 | free((void *)db->logname); 252 | free((void *)db->name); 253 | free(db); 254 | } 255 | 256 | static void detect_handler(struct PgSocket *sk, void *arg, enum PgEvent ev, PGresult *res) 257 | { 258 | int i; 259 | const char *s; 260 | struct List *el, *tmp; 261 | struct PgDatabase *db; 262 | 263 | switch (ev) { 264 | case PGS_CONNECT_OK: 265 | pgs_send_query_simple(sk, "select datname from pg_database" 266 | " where not datistemplate and datallowconn"); 267 | break; 268 | case PGS_RESULT_OK: 269 | /* tag old dbs as dead */ 270 | statlist_for_each(el, &database_list) { 271 | db = container_of(el, struct PgDatabase, head); 272 | db->dropped = true; 273 | } 274 | /* process new dbs */ 275 | for (i = 0; i < PQntuples(res); i++) { 276 | s = PQgetvalue(res, i, 0); 277 | launch_db(s); 278 | } 279 | /* drop old dbs */ 280 | statlist_for_each_safe(el, &database_list, tmp) { 281 | db = container_of(el, struct PgDatabase, head); 282 | if (db->dropped) 283 | drop_db(db, true); 284 | } 285 | pgs_disconnect(sk); 286 | pgs_sleep(sk, cf.check_period); 287 | break; 288 | case PGS_TIMEOUT: 289 | detect_dbs(); 290 | break; 291 | default: 292 | pgs_disconnect(sk); 293 | pgs_sleep(sk, cf.check_period); 294 | break; 295 | } 296 | } 297 | 298 | static void detect_dbs(void) 299 | { 300 | if (!db_template) { 301 | char *cstr = make_connstr(cf.initial_database); 302 | if (!cstr) { 303 | log_error("make_connstr: %s", strerror(errno)); 304 | return; 305 | } 306 | db_template = pgs_create(cstr, detect_handler, NULL, ev_base); 307 | free(cstr); 308 | if (!db_template) { 309 | log_error("pgs_create: %s", strerror(errno)); 310 | return; 311 | } 312 | } 313 | pgs_connect(db_template); 314 | } 315 | 316 | static bool launch_db_cb(void *arg, const char *db) 317 | { 318 | launch_db(db); 319 | return true; 320 | } 321 | 322 | static void recheck_dbs(void) 323 | { 324 | struct PgDatabase *db; 325 | struct List *el, *tmp; 326 | if (cf.database_list && cf.database_list[0]) { 327 | /* tag old dbs as dead */ 328 | statlist_for_each(el, &database_list) { 329 | db = container_of(el, struct PgDatabase, head); 330 | db->dropped = true; 331 | } 332 | /* process new ones */ 333 | if (!parse_word_list(cf.database_list, launch_db_cb, NULL)) { 334 | log_warning("database_list parsing failed: %s", strerror(errno)); 335 | return; 336 | } 337 | /* drop old ones */ 338 | statlist_for_each_safe(el, &database_list, tmp) { 339 | db = container_of(el, struct PgDatabase, head); 340 | if (db->dropped) 341 | drop_db(db, true); 342 | } 343 | 344 | /* done with template for the moment */ 345 | if (db_template) { 346 | pgs_free(db_template); 347 | db_template = NULL; 348 | } 349 | } else if (!db_template) { 350 | log_info("auto-detecting dbs ..."); 351 | detect_dbs(); 352 | } 353 | } 354 | 355 | 356 | static void stats_handler(evutil_socket_t fd, short flags, void *arg) 357 | { 358 | struct timeval tv = { cf.stats_period, 0 }; 359 | 360 | log_info("{ticks: %d, maint: %d, retry: %d}", 361 | stats.n_ticks, stats.n_maint, stats.n_retry); 362 | memset(&stats, 0, sizeof(stats)); 363 | 364 | if (evtimer_add(ev_stats, &tv) < 0) 365 | fatal_perror("evtimer_add"); 366 | } 367 | 368 | static void stats_setup(void) 369 | { 370 | struct timeval tv = { cf.stats_period, 0 }; 371 | ev_stats = evtimer_new(ev_base, stats_handler, NULL); 372 | if (!ev_stats) 373 | fatal_perror("evtimer_new"); 374 | if (evtimer_add(ev_stats, &tv) < 0) 375 | fatal_perror("evtimer_add"); 376 | } 377 | 378 | static void cleanup(void) 379 | { 380 | struct PgDatabase *db; 381 | struct List *elem, *tmp; 382 | 383 | statlist_for_each_safe(elem, &database_list, tmp) { 384 | db = container_of(elem, struct PgDatabase, head); 385 | drop_db(db, false); 386 | } 387 | pgs_free(db_template); 388 | 389 | event_base_free(NULL); 390 | reset_logging(); 391 | 392 | #ifdef SIGHUP 393 | event_free(ev_sighup); 394 | #endif 395 | event_free(ev_sigint); 396 | event_free(ev_sigterm); 397 | event_free(ev_stats); 398 | } 399 | 400 | static void main_loop_once(void) 401 | { 402 | reset_time_cache(); 403 | if (event_base_loop(ev_base, EVLOOP_ONCE) != 0) { 404 | log_error("event_loop failed: %s", strerror(errno)); 405 | } 406 | } 407 | 408 | int main(int argc, char *argv[]) 409 | { 410 | int c; 411 | bool daemon = false; 412 | int sig = 0; 413 | const char *signame = NULL; 414 | 415 | for (c = 1; c < argc; c++) { 416 | if (!strcmp(argv[c], "--ini")) { 417 | printf("%s", sample_ini); 418 | exit(0); 419 | } 420 | if (!strcmp(argv[c], "--help")) { 421 | printf(usage_str); 422 | exit(0); 423 | } 424 | } 425 | 426 | while ((c = getopt(argc, argv, "dqvhVrsk")) != -1) { 427 | switch (c) { 428 | case 'd': 429 | daemon = true; 430 | break; 431 | case 'v': 432 | cf_verbose++; 433 | break; 434 | case 'q': 435 | cf_quiet = 1; 436 | break; 437 | case 'h': 438 | printf(usage_str); 439 | return 0; 440 | case 'V': 441 | printf("%s version %s\n", PACKAGE_NAME, PACKAGE_VERSION); 442 | return 0; 443 | #ifdef SIGHUP 444 | case 'r': 445 | sig = SIGHUP; 446 | signame = "SIGHUP"; 447 | break; 448 | #endif 449 | case 's': 450 | sig = SIGINT; 451 | signame = "SIGINT"; 452 | break; 453 | case 'k': 454 | sig = SIGTERM; 455 | signame = "SIGTERM"; 456 | break; 457 | default: 458 | printf("bad switch: "); 459 | printf(usage_str); 460 | return 1; 461 | } 462 | } 463 | if (optind + 1 != argc) { 464 | fprintf(stderr, "pgqd requires config file\n"); 465 | return 1; 466 | } 467 | 468 | cf.config_file = argv[optind]; 469 | 470 | load_config(); 471 | conf_info.loaded = true; 472 | 473 | if (sig) { 474 | if (!cf.pidfile || !cf.pidfile[0]) { 475 | fprintf(stderr, "No pidfile configured\n"); 476 | return 1; 477 | } 478 | if (signal_pidfile(cf.pidfile, sig)) 479 | fprintf(stderr, "%s sent\n", signame); 480 | else 481 | fprintf(stderr, "Old process is not running\n"); 482 | return 0; 483 | } 484 | 485 | log_info("Starting pgqd " PACKAGE_VERSION); 486 | 487 | daemonize(cf.pidfile, daemon); 488 | 489 | ev_base = event_base_new(); 490 | if (!ev_base) 491 | fatal("event_base_new failed"); 492 | 493 | signal_setup(); 494 | 495 | stats_setup(); 496 | 497 | recheck_dbs(); 498 | 499 | while (!got_sigint) 500 | main_loop_once(); 501 | 502 | cleanup(); 503 | event_base_free(ev_base); 504 | 505 | return 0; 506 | } 507 | -------------------------------------------------------------------------------- /src/pgqd.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __PGQD_H__ 3 | #define __PGQD_H__ 4 | 5 | #include 6 | 7 | #define Assert(x) 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "pgsocket.h" 14 | 15 | enum DbState { 16 | DB_CLOSED, 17 | DB_TICKER_CHECK_PGQ, 18 | DB_TICKER_CHECK_VERSION, 19 | DB_TICKER_RUN, 20 | DB_MAINT_TEST_VERSION, 21 | DB_MAINT_LOAD_OPS, 22 | DB_MAINT_OP, 23 | DB_MAINT_LOAD_QUEUES, 24 | DB_MAINT_ROT1, 25 | DB_MAINT_ROT2, 26 | DB_MAINT_VACUUM_LIST, 27 | DB_MAINT_DO_VACUUM, 28 | }; 29 | 30 | struct MaintOp; 31 | 32 | struct PgDatabase { 33 | struct List head; 34 | const char *name; 35 | const char *logname; 36 | struct PgSocket *c_ticker; 37 | struct PgSocket *c_maint; 38 | struct PgSocket *c_retry; 39 | bool has_pgq; 40 | enum DbState state; 41 | enum DbState maint_state; 42 | bool dropped; 43 | 44 | struct StrList *maint_item_list; 45 | struct StatList maint_op_list; 46 | struct MaintOp *cur_maint; 47 | 48 | bool has_maint_operations; 49 | }; 50 | 51 | struct Config { 52 | const char *config_file; 53 | const char *pidfile; 54 | const char *base_connstr; 55 | const char *initial_database; 56 | const char *database_list; 57 | 58 | double retry_period; 59 | double check_period; 60 | double maint_period; 61 | double ticker_period; 62 | double stats_period; 63 | 64 | double connection_lifetime; 65 | }; 66 | 67 | struct Stats { 68 | int n_ticks; 69 | int n_maint; 70 | int n_retry; 71 | }; 72 | 73 | extern struct Config cf; 74 | extern struct Stats stats; 75 | extern struct event_base *ev_base; 76 | 77 | void launch_ticker(struct PgDatabase *db); 78 | void launch_maint(struct PgDatabase *db); 79 | void launch_retry(struct PgDatabase *db); 80 | 81 | void free_maint(struct PgDatabase *db); 82 | 83 | char *make_connstr(const char *dbname); 84 | 85 | #endif 86 | 87 | -------------------------------------------------------------------------------- /src/pgsocket.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Async Postgres connection. 3 | * 4 | * Copyright (c) 2009 Marko Kreen 5 | * 6 | * Permission to use, copy, modify, and/or distribute this software for any 7 | * purpose with or without fee is hereby granted, provided that the above 8 | * copyright notice and this permission notice appear in all copies. 9 | * 10 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 | */ 18 | 19 | #include "pgsocket.h" 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include 26 | 27 | #define MAX_QRY_ARGS 32 28 | 29 | /* PgSocket.wait_type */ 30 | enum WType { 31 | W_NONE = 0, 32 | W_SOCK, 33 | W_TIME 34 | }; 35 | 36 | typedef void (*libev_cb)(evutil_socket_t sock, short flags, void *arg); 37 | 38 | struct PgSocket { 39 | /* libevent state */ 40 | struct event *ev; 41 | 42 | /* track wait state */ 43 | enum WType wait_type; 44 | 45 | /* EV_READ / EV_WRITE */ 46 | uint8_t wait_event; 47 | 48 | /* should connect after sleep */ 49 | bool reconnect; 50 | 51 | /* current connection */ 52 | PGconn *con; 53 | 54 | /* user handler */ 55 | pgs_handler_f handler_func; 56 | void *handler_arg; 57 | 58 | /* saved connect string */ 59 | const char *connstr; 60 | 61 | /* custom base or NULL */ 62 | struct event_base *base; 63 | 64 | /* temp place for resultset */ 65 | PGresult *last_result; 66 | 67 | usec_t connect_time; 68 | usec_t lifetime; 69 | }; 70 | 71 | /* report event to user callback */ 72 | static void send_event(struct PgSocket *db, enum PgEvent ev) 73 | { 74 | db->handler_func(db, db->handler_arg, ev, NULL); 75 | } 76 | 77 | /* wait socket event from libevent */ 78 | static void wait_event(struct PgSocket *db, short ev, libev_cb fn) 79 | { 80 | int err; 81 | 82 | Assert(!db->wait_type); 83 | 84 | err = event_assign(db->ev, db->base, PQsocket(db->con), ev, fn, db); 85 | if (err < 0) 86 | die("event_assign failed: %s", strerror(errno)); 87 | if (event_add(db->ev, NULL) < 0) 88 | die("event_add failed: %s", strerror(errno)); 89 | 90 | db->wait_type = W_SOCK; 91 | db->wait_event = ev; 92 | } 93 | 94 | /* wait timeout from libevent */ 95 | static void timeout_cb(evutil_socket_t sock, short flags, void *arg) 96 | { 97 | struct PgSocket *db = arg; 98 | 99 | db->wait_type = W_NONE; 100 | 101 | if (db->reconnect) { 102 | db->reconnect = false; 103 | pgs_connect(db); 104 | } else { 105 | send_event(db, PGS_TIMEOUT); 106 | } 107 | } 108 | 109 | /* some error happened */ 110 | static void conn_error(struct PgSocket *db, enum PgEvent ev, const char *desc) 111 | { 112 | log_error("connection error: %s", desc); 113 | log_error("libpq: %s", PQerrorMessage(db->con)); 114 | send_event(db, ev); 115 | } 116 | 117 | /* report previously stored result */ 118 | static void report_last_result(struct PgSocket *db) 119 | { 120 | PGresult *res = db->last_result; 121 | if (!res) 122 | return; 123 | db->last_result = NULL; 124 | 125 | switch (PQresultStatus(res)) { 126 | default: 127 | log_error("%s: %s", PQdb(db->con), PQresultErrorMessage(res)); 128 | /* fallthrough */ 129 | case PGRES_COMMAND_OK: 130 | case PGRES_TUPLES_OK: 131 | case PGRES_COPY_OUT: 132 | case PGRES_COPY_IN: 133 | db->handler_func(db, db->handler_arg, PGS_RESULT_OK, res); 134 | } 135 | PQclear(res); 136 | } 137 | 138 | /* 139 | * Called when select() told that conn is avail for reading. 140 | * 141 | * It should call postgres handlers and then change state if needed. 142 | * 143 | * Because the callback may want to close the connection when processing 144 | * last resultset, the PGresult handover is delayed one step. 145 | */ 146 | static void result_cb(evutil_socket_t sock, short flags, void *arg) 147 | { 148 | struct PgSocket *db = arg; 149 | PGresult *res; 150 | 151 | db->wait_type = W_NONE; 152 | 153 | if (!PQconsumeInput(db->con)) { 154 | conn_error(db, PGS_RESULT_BAD, "PQconsumeInput"); 155 | return; 156 | } 157 | 158 | /* loop until PQgetResult returns NULL */ 159 | while (db->con) { 160 | /* incomplete result? */ 161 | if (PQisBusy(db->con)) { 162 | wait_event(db, EV_READ, result_cb); 163 | return; 164 | } 165 | 166 | /* next result */ 167 | res = PQgetResult(db->con); 168 | if (!res) 169 | break; 170 | 171 | report_last_result(db); 172 | db->last_result = res; 173 | } 174 | 175 | report_last_result(db); 176 | } 177 | 178 | static void flush(struct PgSocket *db); 179 | 180 | static void send_cb(evutil_socket_t sock, short flags, void *arg) 181 | { 182 | struct PgSocket *db = arg; 183 | 184 | db->wait_type = W_NONE; 185 | 186 | flush(db); 187 | } 188 | 189 | /* handle connect states */ 190 | static void connect_cb(evutil_socket_t sock, short flags, void *arg) 191 | { 192 | struct PgSocket *db = arg; 193 | PostgresPollingStatusType poll_res; 194 | 195 | db->wait_type = W_NONE; 196 | 197 | poll_res = PQconnectPoll(db->con); 198 | switch (poll_res) { 199 | case PGRES_POLLING_WRITING: 200 | wait_event(db, EV_WRITE, connect_cb); 201 | break; 202 | case PGRES_POLLING_READING: 203 | wait_event(db, EV_READ, connect_cb); 204 | break; 205 | case PGRES_POLLING_OK: 206 | db->connect_time = get_time_usec(); 207 | send_event(db, PGS_CONNECT_OK); 208 | break; 209 | default: 210 | conn_error(db, PGS_CONNECT_FAILED, "PQconnectPoll"); 211 | } 212 | } 213 | 214 | /* send query to server */ 215 | static void flush(struct PgSocket *db) 216 | { 217 | int res = PQflush(db->con); 218 | if (res > 0) { 219 | wait_event(db, EV_WRITE, send_cb); 220 | } else if (res == 0) { 221 | wait_event(db, EV_READ, result_cb); 222 | } else 223 | conn_error(db, PGS_RESULT_BAD, "PQflush"); 224 | } 225 | 226 | /* override default notice receiver */ 227 | static void custom_notice_receiver(void *arg, const PGresult *res) 228 | { 229 | /* do nothing */ 230 | } 231 | 232 | /* 233 | * Public API 234 | */ 235 | 236 | struct PgSocket *pgs_create(const char *connstr, pgs_handler_f fn, void *handler_arg, struct event_base *base) 237 | { 238 | struct PgSocket *db; 239 | 240 | db = calloc(1, sizeof(*db)); 241 | if (!db) 242 | return NULL; 243 | 244 | db->ev = calloc(1, event_get_struct_event_size()); 245 | if (!db->ev) { 246 | free(db); 247 | return NULL; 248 | } 249 | 250 | db->handler_func = fn; 251 | db->handler_arg = handler_arg; 252 | db->base = base; 253 | 254 | db->connstr = strdup(connstr); 255 | if (!db->connstr) { 256 | pgs_free(db); 257 | return NULL; 258 | } 259 | return db; 260 | } 261 | 262 | void pgs_set_lifetime(struct PgSocket *pgs, double lifetime) 263 | { 264 | pgs->lifetime = USEC * lifetime; 265 | } 266 | 267 | void pgs_connect(struct PgSocket *db) 268 | { 269 | if (db->con) 270 | pgs_disconnect(db); 271 | 272 | db->con = PQconnectStart(db->connstr); 273 | if (db->con == NULL) { 274 | conn_error(db, PGS_CONNECT_FAILED, "PQconnectStart"); 275 | return; 276 | } 277 | 278 | if (PQstatus(db->con) == CONNECTION_BAD) { 279 | conn_error(db, PGS_CONNECT_FAILED, "PQconnectStart"); 280 | return; 281 | } 282 | 283 | PQsetNoticeReceiver(db->con, custom_notice_receiver, db); 284 | 285 | wait_event(db, EV_WRITE, connect_cb); 286 | } 287 | 288 | 289 | void pgs_disconnect(struct PgSocket *db) 290 | { 291 | if (db->wait_type) { 292 | event_del(db->ev); 293 | db->wait_type = W_NONE; 294 | db->reconnect = false; 295 | } 296 | if (db->con) { 297 | PQfinish(db->con); 298 | db->con = NULL; 299 | } 300 | if (db->last_result) { 301 | PQclear(db->last_result); 302 | db->last_result = NULL; 303 | } 304 | } 305 | 306 | void pgs_free(struct PgSocket *db) 307 | { 308 | if (db) { 309 | pgs_disconnect(db); 310 | free((void *)db->connstr); 311 | free(db->ev); 312 | free(db); 313 | } 314 | } 315 | 316 | void pgs_sleep(struct PgSocket *db, double timeout) 317 | { 318 | struct timeval tv; 319 | 320 | Assert(!db->wait_type); 321 | 322 | if (db->con && db->lifetime) { 323 | usec_t now = get_time_usec(); 324 | if (db->connect_time + db->lifetime < now) { 325 | pgs_disconnect(db); 326 | db->reconnect = true; 327 | } 328 | } 329 | 330 | tv.tv_sec = timeout; 331 | tv.tv_usec = (timeout - tv.tv_sec) * USEC; 332 | 333 | evtimer_assign(db->ev, db->base, timeout_cb, db); 334 | if (evtimer_add(db->ev, &tv) < 0) 335 | die("evtimer_add failed: %s", strerror(errno)); 336 | 337 | db->wait_type = W_TIME; 338 | } 339 | 340 | void pgs_reconnect(struct PgSocket *db, double timeout) 341 | { 342 | pgs_disconnect(db); 343 | pgs_sleep(db, timeout); 344 | db->reconnect = true; 345 | } 346 | 347 | void pgs_send_query_simple(struct PgSocket *db, const char *q) 348 | { 349 | int res; 350 | 351 | log_noise("%s", q); 352 | res = PQsendQuery(db->con, q); 353 | if (!res) { 354 | conn_error(db, PGS_RESULT_BAD, "PQsendQuery"); 355 | return; 356 | } 357 | 358 | flush(db); 359 | } 360 | 361 | void pgs_send_query_params(struct PgSocket *db, const char *q, int cnt, ...) 362 | { 363 | int i; 364 | va_list ap; 365 | const char * args[MAX_QRY_ARGS]; 366 | 367 | if (cnt < 0 || cnt > MAX_QRY_ARGS) { 368 | log_warning("bad query arg cnt"); 369 | send_event(db, PGS_RESULT_BAD); 370 | return; 371 | } 372 | 373 | va_start(ap, cnt); 374 | for (i = 0; i < cnt; i++) 375 | args[i] = va_arg(ap, char *); 376 | va_end(ap); 377 | 378 | pgs_send_query_params_list(db, q, cnt, args); 379 | } 380 | 381 | void pgs_send_query_params_list(struct PgSocket *db, const char *q, int cnt, const char *args[]) 382 | { 383 | int res; 384 | 385 | log_noise("%s", q); 386 | res = PQsendQueryParams(db->con, q, cnt, NULL, args, NULL, NULL, 0); 387 | if (!res) { 388 | conn_error(db, PGS_RESULT_BAD, "PQsendQueryParams"); 389 | return; 390 | } 391 | 392 | flush(db); 393 | } 394 | 395 | int pgs_connection_valid(struct PgSocket *db) 396 | { 397 | return (db->con != NULL); 398 | } 399 | 400 | PGconn *pgs_get_connection(struct PgSocket *db) 401 | { 402 | return db->con; 403 | } 404 | 405 | bool pgs_waiting_for_reply(struct PgSocket *db) 406 | { 407 | if (!db->con) 408 | return false; 409 | if (PQstatus(db->con) != CONNECTION_OK) 410 | return false; 411 | return (db->wait_type == W_SOCK) && (db->wait_event == EV_READ); 412 | } 413 | -------------------------------------------------------------------------------- /src/pgsocket.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2009 Marko Kreen 3 | * 4 | * Permission to use, copy, modify, and/or distribute this software for any 5 | * purpose with or without fee is hereby granted, provided that the above 6 | * copyright notice and this permission notice appear in all copies. 7 | * 8 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 11 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 14 | * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 | */ 16 | 17 | /** @file 18 | * 19 | * Async Postgres connection framework. 20 | */ 21 | #ifndef _USUAL_PGSOCKET_H_ 22 | #define _USUAL_PGSOCKET_H_ 23 | 24 | #include 25 | 26 | #include 27 | 28 | /** 29 | * Event types reported to user handler function. 30 | */ 31 | enum PgEvent { 32 | /** Connection establishing finished */ 33 | PGS_CONNECT_OK, 34 | /** Connection establishing failed */ 35 | PGS_CONNECT_FAILED, 36 | /** Got result from query either resultset or DB error */ 37 | PGS_RESULT_OK, 38 | /** Query execution failed */ 39 | PGS_RESULT_BAD, 40 | /** Wakeup from timed sleep */ 41 | PGS_TIMEOUT, 42 | }; 43 | 44 | struct PgSocket; 45 | struct event_base; 46 | 47 | typedef void (*pgs_handler_f)(struct PgSocket *pgs, void *arg, enum PgEvent dbev, PGresult *res); 48 | 49 | /** Create PgSocket. 50 | * 51 | * It does not launch connection yet, use \ref pgs_connect() for that. 52 | * 53 | * @param connstr libpq connect string 54 | * @param fn callback function for event handling 55 | * @param arg extra context for callback 56 | * @return Initialized PgSocket structure 57 | */ 58 | struct PgSocket *pgs_create(const char *connstr, pgs_handler_f fn, void *arg, struct event_base *base); 59 | 60 | /** Release PgSocket */ 61 | void pgs_free(struct PgSocket *db); 62 | 63 | /** Set connection lifetime (in seconds) */ 64 | void pgs_set_lifetime(struct PgSocket *pgs, double lifetime); 65 | 66 | /** Launch connection */ 67 | void pgs_connect(struct PgSocket *db); 68 | 69 | /** Drop connection */ 70 | void pgs_disconnect(struct PgSocket *db); 71 | 72 | /** Send simple query */ 73 | void pgs_send_query_simple(struct PgSocket *db, const char *query); 74 | 75 | /** Send extended query, args from varargs */ 76 | void pgs_send_query_params(struct PgSocket *db, const char *query, int nargs, ...); 77 | 78 | /** Send extended query, args from list */ 79 | void pgs_send_query_params_list(struct PgSocket *db, const char *query, int nargs, const char *argv[]); 80 | 81 | /** Ignore the connection for specified time */ 82 | void pgs_sleep(struct PgSocket *db, double timeout); 83 | 84 | /** Disconnect, sleep, reconnect */ 85 | void pgs_reconnect(struct PgSocket *db, double timeout); 86 | 87 | /** Does PgSocket have established connection */ 88 | int pgs_connection_valid(struct PgSocket *db); 89 | 90 | /** Return underlying Postgres connection */ 91 | PGconn *pgs_get_connection(struct PgSocket *db); 92 | 93 | bool pgs_waiting_for_reply(struct PgSocket *db); 94 | 95 | #endif 96 | -------------------------------------------------------------------------------- /src/retry.c: -------------------------------------------------------------------------------- 1 | 2 | #include "pgqd.h" 3 | 4 | 5 | static void close_retry(struct PgDatabase *db, double sleep_time) 6 | { 7 | log_debug("%s: close_retry, %f", db->logname, sleep_time); 8 | pgs_reconnect(db->c_retry, sleep_time); 9 | } 10 | 11 | static void run_retry(struct PgDatabase *db) 12 | { 13 | const char *q = "select * from pgq.maint_retry_events()"; 14 | log_debug("%s: %s", db->logname, q); 15 | pgs_send_query_simple(db->c_retry, q); 16 | } 17 | 18 | static void parse_retry(struct PgDatabase *db, PGresult *res) 19 | { 20 | if (PQntuples(res) == 1) { 21 | char *val = PQgetvalue(res, 0, 0); 22 | stats.n_retry += atoi(val); 23 | if (strcmp(val, "0") != 0) { 24 | run_retry(db); 25 | return; 26 | } 27 | } 28 | close_retry(db, cf.retry_period); 29 | } 30 | 31 | static void retry_handler(struct PgSocket *s, void *arg, enum PgEvent ev, PGresult *res) 32 | { 33 | struct PgDatabase *db = arg; 34 | 35 | switch (ev) { 36 | case PGS_CONNECT_OK: 37 | log_debug("%s: starting retry event processing", db->logname); 38 | run_retry(db); 39 | break; 40 | case PGS_RESULT_OK: 41 | if (PQresultStatus(res) != PGRES_TUPLES_OK) 42 | close_retry(db, 20); 43 | else 44 | parse_retry(db, res); 45 | break; 46 | case PGS_TIMEOUT: 47 | log_debug("%s: retry timeout", db->logname); 48 | launch_retry(db); 49 | break; 50 | default: 51 | log_warning("%s: default reconnect", db->logname); 52 | pgs_reconnect(db->c_retry, 30); 53 | break; 54 | } 55 | } 56 | 57 | void launch_retry(struct PgDatabase *db) 58 | { 59 | char *cstr; 60 | if (db->c_retry) { 61 | log_debug("%s: retry already initialized", db->logname); 62 | } else { 63 | log_debug("%s: launch_retry", db->logname); 64 | cstr = make_connstr(db->name); 65 | if (!cstr) { 66 | log_error("make_connstr: %s", strerror(errno)); 67 | return; 68 | } 69 | db->c_retry = pgs_create(cstr, retry_handler, db, ev_base); 70 | free(cstr); 71 | if (!db->c_retry) { 72 | log_error("pgs_create: %s", strerror(errno)); 73 | return; 74 | } 75 | } 76 | pgs_connect(db->c_retry); 77 | } 78 | 79 | -------------------------------------------------------------------------------- /src/ticker.c: -------------------------------------------------------------------------------- 1 | #include "pgqd.h" 2 | 3 | static void run_pgq_check(struct PgDatabase *db) 4 | { 5 | const char *q = "select 1 from pg_catalog.pg_namespace where nspname='pgq'"; 6 | log_debug("%s: %s", db->logname, q); 7 | pgs_send_query_simple(db->c_ticker, q); 8 | db->state = DB_TICKER_CHECK_PGQ; 9 | } 10 | 11 | static void run_version_check(struct PgDatabase *db) 12 | { 13 | const char *q = "select pgq.version()"; 14 | log_debug("%s: %s", db->logname, q); 15 | pgs_send_query_simple(db->c_ticker, q); 16 | db->state = DB_TICKER_CHECK_VERSION; 17 | } 18 | 19 | static void run_ticker(struct PgDatabase *db) 20 | { 21 | const char *q = "select pgq.ticker()"; 22 | log_noise("%s: %s", db->logname, q); 23 | pgs_send_query_simple(db->c_ticker, q); 24 | db->state = DB_TICKER_RUN; 25 | } 26 | 27 | static void close_ticker(struct PgDatabase *db, double sleep_time) 28 | { 29 | log_debug("%s: close_ticker, %f", db->logname, sleep_time); 30 | db->state = DB_CLOSED; 31 | pgs_reconnect(db->c_ticker, sleep_time); 32 | } 33 | 34 | static void parse_pgq_check(struct PgDatabase *db, PGresult *res) 35 | { 36 | db->has_pgq = PQntuples(res) == 1; 37 | 38 | if (!db->has_pgq) { 39 | log_debug("%s: no pgq", db->logname); 40 | close_ticker(db, cf.check_period); 41 | } else { 42 | run_version_check(db); 43 | } 44 | } 45 | 46 | static void parse_version_check(struct PgDatabase *db, PGresult *res) 47 | { 48 | char *ver; 49 | if (PQntuples(res) != 1) { 50 | log_debug("%s: calling pgq.version() failed", db->logname); 51 | goto badpgq; 52 | } 53 | ver = PQgetvalue(res, 0, 0); 54 | if (ver[0] < '3') { 55 | log_debug("%s: bad pgq version: %s", db->logname, ver); 56 | goto badpgq; 57 | } 58 | log_info("%s: pgq version ok: %s", db->logname, ver); 59 | 60 | run_ticker(db); 61 | if (!db->c_maint) 62 | launch_maint(db); 63 | if (!db->c_retry) 64 | launch_retry(db); 65 | return; 66 | 67 | badpgq: 68 | db->has_pgq = false; 69 | log_info("%s: bad pgq version, ignoring", db->logname); 70 | close_ticker(db, cf.check_period); 71 | } 72 | 73 | static void parse_ticker_result(struct PgDatabase *db, PGresult *res) 74 | { 75 | if (PQntuples(res) != 1) { 76 | log_debug("%s: calling pgq.ticker() failed", db->logname); 77 | } else { 78 | stats.n_ticks++; 79 | } 80 | 81 | pgs_sleep(db->c_ticker, cf.ticker_period); 82 | } 83 | 84 | static void tick_handler(struct PgSocket *s, void *arg, enum PgEvent ev, PGresult *res) 85 | { 86 | struct PgDatabase *db = arg; 87 | ExecStatusType st; 88 | 89 | switch (ev) { 90 | case PGS_CONNECT_OK: 91 | run_pgq_check(db); 92 | break; 93 | case PGS_RESULT_OK: 94 | if (PQresultStatus(res) != PGRES_TUPLES_OK) { 95 | close_ticker(db, 10); 96 | break; 97 | } 98 | switch (db->state) { 99 | case DB_TICKER_CHECK_PGQ: 100 | parse_pgq_check(db, res); 101 | break; 102 | case DB_TICKER_CHECK_VERSION: 103 | parse_version_check(db, res); 104 | break; 105 | case DB_TICKER_RUN: 106 | parse_ticker_result(db, res); 107 | break; 108 | case DB_CLOSED: 109 | st = PQresultStatus(res); 110 | log_warning("%s: Weird state: RESULT_OK + DB_CLOSED (%s)", 111 | db->logname, PQresStatus(st)); 112 | close_ticker(db, 10); 113 | break; 114 | default: 115 | log_warning("%s: bad state: %d", db->logname, db->state); 116 | close_ticker(db, 10); 117 | break; 118 | } 119 | break; 120 | case PGS_TIMEOUT: 121 | log_noise("%s: tick timeout", db->logname); 122 | if (!pgs_connection_valid(db->c_ticker)) 123 | launch_ticker(db); 124 | else 125 | run_ticker(db); 126 | break; 127 | default: 128 | log_warning("%s: default timeout", db->logname); 129 | pgs_reconnect(db->c_ticker, 60); 130 | break; 131 | } 132 | } 133 | 134 | void launch_ticker(struct PgDatabase *db) 135 | { 136 | log_debug("%s: launch_ticker", db->logname); 137 | if (!db->c_ticker) { 138 | char *cstr = make_connstr(db->name); 139 | if (!cstr) { 140 | log_error("make_connstr: %s", strerror(errno)); 141 | return; 142 | } 143 | db->c_ticker = pgs_create(cstr, tick_handler, db, ev_base); 144 | free(cstr); 145 | if (!db->c_ticker) { 146 | log_error("pgs_create: %s", strerror(errno)); 147 | return; 148 | } 149 | pgs_set_lifetime(db->c_ticker, cf.connection_lifetime); 150 | } 151 | pgs_connect(db->c_ticker); 152 | } 153 | 154 | -------------------------------------------------------------------------------- /tests/test.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | cd $(dirname $0) 4 | 5 | PGQD=../pgqd 6 | 7 | export PGHOST="${PGHOST:-127.0.0.1}" 8 | export PGPORT="${PGPORT:-5432}" 9 | export LANG=C 10 | export LC_ALL=C 11 | 12 | LOGDIR=log 13 | 14 | set -o pipefail 15 | 16 | mkdir -p log 17 | rm -f log/* 18 | 19 | ulimit -c unlimited 20 | 21 | echo "Creating databases" 22 | for dbname in db1; do 23 | dropdb --if-exists $dbname 24 | createdb $dbname || exit 1 25 | psql -q -d $dbname -c "create extension pgq" 26 | done 27 | 28 | die() { 29 | echo $@ 30 | exit 1 31 | } 32 | 33 | runtest() { 34 | local status 35 | 36 | printf "`date` running $1 ... " 37 | conf="${LOGDIR}/$1.ini" 38 | logfile="${LOGDIR}/$1.log" 39 | outfile="${LOGDIR}/$1.out" 40 | pidfile="${LOGDIR}/$1.pid" 41 | printf "[pgqd]\nlogfile=${logfile}\npidfile=${pidfile}\ncheck_period=3\nmaint_period=7\n" > "${conf}" 42 | eval "$1" "${conf}" "${logfile}" > ${outfile} 2>&1 43 | res=$? 44 | date >> ${outfile} 45 | if [ $res -eq 0 ]; then 46 | echo "ok" 47 | else 48 | echo "FAILED" 49 | cat ${outfile} | sed 's/^/out> /' 50 | if test -f "${logfile}"; then 51 | cat ${logfile} | sed 's/^/log> /' 52 | fi 53 | fi 54 | 55 | test -f "${pidfile}" && kill $(cat "${pidfile}") 56 | 57 | return $res 58 | } 59 | 60 | # 61 | # testcases 62 | # 63 | 64 | test_version() { 65 | ln=$(${PGQD} -V) 66 | case "$ln" in 67 | *version*) res=0;; 68 | *) res=1;; 69 | esac 70 | return $res 71 | } 72 | 73 | test_show_ini() { 74 | ${PGQD} --ini | grep -q 'logfile' 75 | return $? 76 | } 77 | 78 | test_stop() { 79 | ${PGQD} -d $1 || return 1 80 | sleep 1 81 | ${PGQD} -s $1 || return 1 82 | sleep 1 83 | grep -q SIGINT $2 84 | return $? 85 | } 86 | 87 | test_reload() { 88 | ${PGQD} -d $1 || return 1 89 | sleep 1 90 | ${PGQD} -r $1 || return 1 91 | sleep 1 92 | grep -q SIGHUP $2 93 | res=$? 94 | ${PGQD} -s $1 || return 1 95 | sleep 1 96 | return $res 97 | } 98 | 99 | create_queue() { 100 | db="$1" 101 | qname="$2" 102 | psql -q -d "${db}" -c "select pgq.create_queue('${qname}')" \ 103 | || die "queue creation failed" 104 | psql -q -d "${db}" -c "update pgq.queue set queue_rotation_period='10 seconds' where queue_name='${qname}'" \ 105 | || die "queue setup failed" 106 | } 107 | 108 | test_ticker() { 109 | db="db1" 110 | qname="test1" 111 | create_queue "${db}" "${qname}" 112 | ${PGQD} -d "$1" || return 1 113 | sleep 3 114 | tick_id=$(psql -q -d "${db}" -At -c "select last_tick_id from pgq.get_queue_info('${qname}')") 115 | echo "tick_id=${tick_id}" 116 | test "${tick_id}" -gt 2 || return 1 117 | return 0 118 | } 119 | 120 | test_maint() { 121 | db="db1" 122 | qname="test2" 123 | create_queue "${db}" "${qname}" 124 | ${PGQD} -d "$1" || return 1 125 | sleep 15 126 | cur_tbl=$(psql -q -d "${db}" -At -c "select queue_cur_table from pgq.get_queue_info('${qname}')") 127 | echo "cur_tbl=${cur_tbl}" 128 | test "${cur_tbl}" -gt 0 || return 1 129 | return 0 130 | } 131 | 132 | testlist=" 133 | test_version 134 | test_show_ini 135 | test_stop 136 | test_reload 137 | test_ticker 138 | test_maint 139 | " 140 | 141 | if [ $# -gt 0 ]; then 142 | testlist="$*" 143 | fi 144 | 145 | final=0 146 | for testcase in $testlist; do 147 | runtest $testcase || final=1 148 | done 149 | 150 | exit $final 151 | 152 | --------------------------------------------------------------------------------