├── .dockerignore ├── .gitattributes ├── .gitignore ├── Changelog.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── TODO-ru.md ├── VERSION ├── Vagrantfile ├── change-size-vm-disk.md ├── circle.yml ├── docker-compose.yml ├── entrypoint.sh ├── filebeat ├── Dockerfile └── config │ ├── filebeat.yml │ └── prospectors.d │ └── default.yml ├── logstash ├── Dockerfile ├── config │ └── logstash.yml └── pipeline │ └── logstash.conf ├── mamonsu ├── Dockerfile ├── README.md ├── agent.conf ├── init-start.sh └── mamonsu-entrypoint.sh ├── pgbadger ├── Dockerfile ├── nginx.conf ├── start-pgbadger-cron ├── start_analyze.sh └── supervisord.conf ├── powa-web ├── Dockerfile └── powa-web.conf ├── psql.bat ├── psql.sh ├── runtime ├── env-defaults ├── functions ├── plsql │ └── make_tablespaces.pgsql └── warmup ├── tools ├── compsess-status.os └── postgrepinning └── upgrade.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | circle.yml 3 | LICENSE 4 | VERSION 5 | README.md 6 | Changelog.md 7 | Makefile 8 | powa-web 9 | docker-compose.yml 10 | TODO-ru.md 11 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.sh -crlf 2 | Dockerfile -crlf 3 | runtime/env-defaults -crlf 4 | runtime/functions -crlf 5 | runtime/warmup -crlf 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | -------------------------------------------------------------------------------- /Changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | **9.6.5-5** 4 | 5 | * extend docker-compose with pghero, mamonsu agent and docker-volumes 6 | * powa now may use master as monitor host 7 | * create tablespaces in postgresql with 1C support of v8index and v8data tablespaces 8 | * fix the function what sets the max connection params 9 | * set the replica commit async (sync=off) 10 | 11 | **9.6.5-4** 12 | 13 | - add mamonsu tune by default 14 | - fix CRLF 15 | - add pg_repack extensions 16 | - fix deprecated Docker label 17 | - use circleci to CICD 18 | 19 | **9.6-2.1** 20 | 21 | - add extensions hypopg and powa by default with building from source 22 | - migrate to postgresql-pro images 23 | - add official pgadmin 4-2 docker compose 24 | - add repication cluster example to docker compose 25 | - add powa-web to analyze from replica 26 | - create official image https://hub.docker.com/r/silverbulleters/ya-docker-postgresql-1c/ 27 | - check the build and work on linux and windows 28 | 29 | **9.6** 30 | - postgresql: upgrade to 9.6 31 | - upgrade baseimage to sameersbn/ubuntu:14.04.20170110 32 | 33 | **9.5** 34 | - postgresql: upgrade to 9.5 35 | 36 | **9.4-17** 37 | - added `DB_EXTENSION` configuration parameter 38 | 39 | **9.4-12** 40 | - removed use of single-user mode 41 | - added `DB_TEMPLATE` variable to specify the database template 42 | 43 | **9.4-11** 44 | - added `PG_PASSWORD` variable to specify password for `postgres` user 45 | 46 | **9.4-9** 47 | - complete rewrite 48 | - `PSQL_TRUST_LOCALNET` config parameter renamed to `PG_TRUST_LOCALNET` 49 | - `PSQL_MODE` config parameter renamed to `REPLICATION_MODE` 50 | - `PSQL_SSLMODE` config parameter renamed to `REPLICATION_SSLMODE` 51 | - defined `/etc/postgresql/certs` as the mountpoint to install SSL key and certificate 52 | - added `PG_SSL` parameter to enable/disable SSL support 53 | - `DB_LOCALE` config parameter renamed to `PG_LOCALE` 54 | - complete rewrite of the README 55 | - add support for creating backups using `pg_basebackup` 56 | - removed `PG_LOCALE` option (doesn't work!) 57 | - added `DEBUG` option to enable bash debugging 58 | 59 | **9.4-2** 60 | - added replication options 61 | 62 | **9.4-1** 63 | - start: removed `pwfile` logic 64 | - init: added `USERMAP_*` configuration options 65 | - base image update to fix SSL vulnerability 66 | 67 | **9.4** 68 | - postgresql: upgrade to 9.4 69 | 70 | **9.1-2** 71 | - use the official postgresql apt repo 72 | - feature: automatic data migration on upgrade 73 | 74 | **9.1-1** 75 | - upgrade to sameersbn/ubuntu:20141001, fixes shellshock 76 | - support creation of users and databases at launch (`docker run`) 77 | - mount volume at `/var/run/postgresql` allowing the postgresql unix socket to be exposed 78 | 79 | **9.1** 80 | - optimized image size by removing `/var/lib/apt/lists/*`. 81 | - update to the sameersbn/ubuntu:12.04.20140818 baseimage 82 | - removed use of supervisord 83 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM sameersbn/ubuntu:14.04.20170123 2 | 3 | LABEL maintainer.base="sameer@damagehead.com" \ 4 | maintainer.current="team@silverbulleters.org" 5 | 6 | ENV DEBIAN_FRONTEND=noninteractive 7 | 8 | ENV PG_APP_HOME="/etc/docker-postgresql"\ 9 | PG_VERSION=9.6 \ 10 | PG_USER=postgres \ 11 | PG_HOME=/var/lib/postgresql \ 12 | PG_RUNDIR=/run/postgresql \ 13 | PG_LOGDIR=/var/log/postgresql \ 14 | PG_CERTDIR=/etc/postgresql/certs 15 | 16 | ENV PG_BINDIR=/usr/lib/postgresql/${PG_VERSION}/bin \ 17 | PG_DATADIR=${PG_HOME}/${PG_VERSION}/main \ 18 | PG_WAL=${PG_HOME}/pg_xlog \ 19 | PG_TEMPTBLSPC=${PG_HOME}/temptblspc \ 20 | PG_V81C_DATA=${PG_HOME}/v81c_data \ 21 | PG_V81C_INDEX=${PG_HOME}/v81c_index 22 | 23 | RUN apt-get update && apt-get install -y locales \ 24 | && localedef -i ru_RU -c -f UTF-8 -A /usr/share/locale/locale.alias ru_RU.UTF-8 \ 25 | && update-locale LANG=ru_RU.UTF-8 26 | 27 | #TODO add en_US locale 28 | ENV LANG ru_RU.UTF-8 29 | ENV LC_MESSAGES "POSIX" 30 | 31 | ADD tools/postgrepinning /etc/apt/preferences.d/postgres 32 | 33 | RUN wget --quiet -O - http://1c.postgrespro.ru/keys/GPG-KEY-POSTGRESPRO-1C | apt-key add - \ 34 | && echo 'deb http://1c.postgrespro.ru/deb/ trusty main' > /etc/apt/sources.list.d/postgrespro-1c.list \ 35 | && apt-get update \ 36 | && DEBIAN_FRONTEND=noninteractive apt-get install -y curl acl \ 37 | postgresql-pro-1c-${PG_VERSION} postgresql-client-pro-1c-${PG_VERSION} postgresql-contrib-pro-1c-${PG_VERSION} \ 38 | && ln -sf ${PG_DATADIR}/postgresql.conf /etc/postgresql/${PG_VERSION}/main/postgresql.conf \ 39 | && ln -sf ${PG_DATADIR}/pg_hba.conf /etc/postgresql/${PG_VERSION}/main/pg_hba.conf \ 40 | && ln -sf ${PG_DATADIR}/pg_ident.conf /etc/postgresql/${PG_VERSION}/main/pg_ident.conf 41 | 42 | WORKDIR /tmp 43 | 44 | RUN curl -s https://packagecloud.io/install/repositories/postgrespro/mamonsu/script.deb.sh | bash 45 | RUN apt-get install mamonsu -y 46 | 47 | WORKDIR /usr/local/src 48 | 49 | RUN apt-get update && apt-get install -y \ 50 | gcc \ 51 | jq \ 52 | make \ 53 | postgresql-contrib-pro-1c-${PG_VERSION} \ 54 | postgresql-server-dev-pro-1c-${PG_VERSION} \ 55 | postgresql-plpython-pro-1c-${PG_VERSION} \ 56 | && wget -O- $(wget -O- https://api.github.com/repos/dalibo/powa-archivist/releases/latest|jq -r '.tarball_url') | tar -xzf - \ 57 | && wget -O- $(wget -O- https://api.github.com/repos/dalibo/pg_qualstats/releases/latest|jq -r '.tarball_url') | tar -xzf - \ 58 | && wget -O- $(wget -O- https://api.github.com/repos/dalibo/pg_stat_kcache/releases/latest|jq -r '.tarball_url') | tar -xzf - \ 59 | && wget -O- $(wget -O- https://api.github.com/repos/dalibo/hypopg/releases/latest|jq -r '.tarball_url') | tar -xzf - \ 60 | && wget -O- $(wget -O- https://api.github.com/repos/rjuju/pg_track_settings/releases/latest|jq -r '.tarball_url') | tar -xzf - \ 61 | && wget -O- $(wget -O- https://api.github.com/repos/reorg/pg_repack/tags|jq -r '.[0].tarball_url') | tar -xzf - \ 62 | && for f in $(ls); do cd $f; make install; cd ..; rm -rf $f; done \ 63 | && apt-get purge -y --auto-remove curl gcc jq make postgresql-server-dev-pro-1c-${PG_VERSION} wget 64 | 65 | RUN rm -rf ${PG_HOME} \ 66 | && rm -rf /var/lib/apt/lists/* 67 | 68 | COPY runtime/ ${PG_APP_HOME}/ 69 | COPY entrypoint.sh /sbin/entrypoint.sh 70 | RUN chmod 755 /sbin/entrypoint.sh 71 | 72 | EXPOSE 5432/tcp 73 | VOLUME ["${PG_HOME}", "${PG_RUNDIR}", "${PG_LOGDIR}", "${PG_DATADIR}"] 74 | VOLUME ["${PG_TEMPTBLSPC}", "${PG_V81C_DATA}", "${PG_V81C_INDEX}"] 75 | WORKDIR ${PG_HOME} 76 | ENTRYPOINT ["/sbin/entrypoint.sh"] 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Sameer Naik 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: build 2 | 3 | build: 4 | @docker build --tag=silverbulleters/ya-docker-postgresql-1c . 5 | 6 | release: build 7 | @docker build --tag=silverbulleters/ya-docker-postgresql-1c:$(shell cat VERSION) . 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # VanessaDockers/postgresql:9.6.5-5 with 1C support 2 | 3 | Stable Build Status | Experimental Build Status | 4 | :-------------------:|:----------------------:| 5 | [![CircleCI](https://circleci.com/gh/VanessaDockers/ya-docker-postgresql-1C/tree/master.svg?style=svg)](https://circleci.com/gh/VanessaDockers/ya-docker-postgresql-1C/tree/master) | [![CircleCI](https://circleci.com/gh/VanessaDockers/ya-docker-postgresql-1C/tree/experimental.svg?style=svg)](https://circleci.com/gh/VanessaDockers/ya-docker-postgresql-1C/tree/experimental) | 6 | 7 | - [Введение](#введение) 8 | - [Доработка](#доработка) 9 | - [Обсуждения](#обсуждения) 10 | - [Начиная использование](#приступая-к-работе) 11 | - [Установка](#установка) 12 | - [Быстрый старт](#быстрый-старт) 13 | - [Персистентность](#персистентность) 14 | - [Trusting local connections](#trusting-local-connections) 15 | - [Setting `postgres` user password](#setting-postgres-user-password) 16 | - [Creating database user](#creating-database-user) 17 | - [Creating databases](#creating-databases) 18 | - [Granting user access to a database](#granting-user-access-to-a-database) 19 | - [Enabling extensions](#enabling-extensions) 20 | - [Creating replication user](#creating-replication-user) 21 | - [Setting up a replication cluster](#setting-up-a-replication-cluster) 22 | - [Creating a snapshot](#creating-a-snapshot) 23 | - [Creating a backup](#creating-a-backup) 24 | - [Command-line arguments](#command-line-arguments) 25 | - [Logs](#logs) 26 | - [UID/GID mapping](#uidgid-mapping) 27 | - [Maintenance](#maintenance) 28 | - [Upgrading](#upgrading) 29 | - [Shell Access](#shell-access) 30 | 31 | # Введение 32 | 33 | `Dockerfile` для создания образа [Docker](https://www.docker.com/) контейнера [PostgreSQL Pro](https://postgrespro.ru/). 34 | 35 | PostgreSQL это объектно ориентированая система управления базами данных с акцентом на расширяемость и соответствие стандартам [[источник](https://ru.wikipedia.org/wiki/PostgreSQL)]. 36 | 37 | ## Доработка 38 | 39 | Для доработки данного образа используйте концепцию `fork` и `pull-request` 40 | 41 | ## Обсуждения 42 | 43 | Прежде чем принимать решения об обсуждении проблемы, попробуйте обновить `Docker` до последней версии и проверьте не исправило ли это вашу проблему. Ознакомьтесь c инструкцией [по установке и обновлению Docker](https://docs.docker.com/installation) 44 | 45 | Пользователи SELinux должны попробовать воспользоваться командой `setenforce 0` чтобы удостовериться в исправлении проблемы. 46 | 47 | Если вышеуказанные рекомендации не помогают, тогда произведите исследование, исправьте проблемы и сформируйте `pull request` 48 | при формировании `pull request'а` пожалуйста отразите 49 | 50 | - вывод команд `docker version` и `docker info` 51 | - проверяли ли вы работу в режиме `docker run` или использовали файл `docker-compose.yml` 52 | - использовали ли в момент проверки [Boot2Docker](http://www.boot2docker.io), [VirtualBox](https://www.virtualbox.org), и т.д. 53 | 54 | Дополнительно для данного проекта [открыт чат gitter](https://gitter.im/VanessaDockers/pgsteroids) 55 | 56 | # Приступая к работе 57 | 58 | ## Установка 59 | 60 | Автоматические сборки данного образа доступна через хаб [Dockerhub](https://hub.docker.com/r//silverbulleters/ya-docker-postgresql-1c) и являются рекомендованным способом установки 61 | 62 | 63 | ```bash 64 | docker pull silverbulleters/ya-docker-postgresql-1c 65 | ``` 66 | 67 | Конечно же вы можете собрать образ и сами. 68 | 69 | ```bash 70 | docker build -t silverbulleters/ya-docker-postgresql-1c github.com/VanessDockers/ya-docker-postgresql-1c 71 | ``` 72 | 73 | ## Быстрый старт 74 | 75 | Для запуска просто запустите команду: 76 | 77 | ```bash 78 | docker run --name postgresql -itd --restart always \ 79 | --publish 5432:5432 \ 80 | --volume /srv/docker/postgresql:/var/lib/postgresql \ 81 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 82 | ``` 83 | 84 | Для подключения к сервер запустите команду: 85 | 86 | ```bash 87 | docker exec -it postgresql sudo -u postgres psql 88 | ``` 89 | 90 | 91 | Примерные шаги по запуску всего контура целиком: 92 | - Получить (pull) или собрать (build) образы сервисов 93 | - Запустить сервис коллектора журналов - logstash 94 | - Запустить сервис PostgreSQL (мастер) 95 | - При необходимости запустить реплику 96 | - Запустить сервис передачи журналов работы (log-beats) 97 | - Запустить сервис отображения отчетов по журналам работы pgbadger 98 | - Запустить сервис Pgadmin 99 | - Создать базу (если база для 1с, то только средствами 1С: консоль администрирования или стартовое окно Предприятия) 100 | - На созданную базу подключить сервис Pghero (редактировать docker-compose.yml) 101 | - Запустить сервис POWA 102 | - Запуск службы агента мониторинга состоит из двух частей: 103 | - предварительный запуск: создание и наполнение служебных таблиц и шаблонов на хосте мониторинга 104 | - рабочий запуск 105 | 106 | 107 | 108 | *Дополнительно вы можете использовать примерный файл [docker-compose.yml](docker-compose.yml) для запуска вашего контейнера с помощью [Docker Compose](https://docs.docker.com/compose/). Обратите внимание: для вашего удобства созданы 2 файла `psql.bat` и `psql.sh` которые позволяют сразу же войти в режим `psql` с учетом того что вы склонировали репозиторий в каталог ya-docker-postgresql-1c* 109 | 110 | ## Сохранение данных контейнеров 111 | 112 | Для хранения данных служб в контейнерах определены несколько томов: том для данных субд, отдельный том для табличных пространств (временные таблицы, данные платформы 1С:Предприятие, индексы платформы 1С:Предприятие), том для хранения журналов работы субд. 113 | 114 | Чтобы службы сохраняли своё состояние при пересоздании контейнеров служб необходимо смонтировать тома в соответующие расположения. 115 | 116 | Для корректной работы как на платформе windows, так и на платформе linux в качестве точек монтирования были определены отдельные контейнеры для хранения данных (docker volumes). 117 | 118 | Для загрузки большого объема данных в окружении разработчика (docker for windows или docker toolbox) необходимо увеличить размер диска виртуальной машины. Для этого нужно выполнить соответствующие инструкции из статьи [Изменение размера диска виртуальной машины](change-size-vm-disk.md) 119 | 120 | При запуске на продуктивной среде можно дополнительно указать точки монтирования куда будут подключены тома хранения данных на докер хосте. 121 | 122 | > *В [Быстром старте](#быстрый-старт) команда уже монтирует точку подключения как персистентную.* 123 | 124 | Пользователи `SELinux` должны обновить контекст безопасности в точке подключения, чтобы корректно использовать Docker: 125 | 126 | ```bash 127 | mkdir -p /srv/docker/postgresql 128 | chcon -Rt svirt_sandbox_file_t /srv/docker/postgresql 129 | ``` 130 | 131 | > **Примечание** 132 | > 133 | > обратите внимание, в текущей версии docker-compose данные PostgreSQL размещаются в именованных томах (docker volumes), чтобы это изменить, вам необходимо явно переопределить точки монтирования. 134 | 135 | ## Доверительные локальные соединения 136 | 137 | По умолчанию, соединения к PostgreSQL серверу требуют аутентификация при помощи пароля. Если вы решите использовать доверительные соединения без пароля из локальной сети, вы должны указать переменную окружения `PG_TRUST_LOCALNET` 138 | 139 | ```bash 140 | docker run --name postgresql -itd --restart always \ 141 | --env 'PG_TRUST_LOCALNET=true' \ 142 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 143 | ``` 144 | 145 | > **Примечание** 146 | > 147 | > The local network here is network to which the container is attached. This has different meanings depending on the `--net` parameter specified while starting the container. In the default configuration, this parameter would trust connections from other containers on the `docker0` bridge. 148 | 149 | ## Setting `postgres` user password 150 | 151 | By default the `postgres` user is not assigned a password and as a result you can only login to the PostgreSQL server locally. If you wish to login remotely to the PostgreSQL server as the `postgres` user, you will need to assign a password for the user using the `PG_PASSWORD` variable. 152 | 153 | ```bash 154 | docker run --name postgresql -itd --restart always \ 155 | --env 'PG_PASSWORD=passw0rd' \ 156 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 157 | ``` 158 | 159 | 160 | > **Примечание** 161 | > 162 | > - When [persistence](#persistence) is in use, `PG_PASSWORD` is effective on the first run. 163 | > - This feature is only available in the `latest` and versions > `9.4-10` 164 | 165 | ## Creating database user 166 | 167 | A new PostgreSQL database user can be created by specifying the `DB_USER` and `DB_PASS` variables while starting the container. 168 | 169 | ```bash 170 | docker run --name postgresql -itd --restart always \ 171 | --env 'DB_USER=dbuser' --env 'DB_PASS=dbuserpass' \ 172 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 173 | ``` 174 | 175 | > **Примечания** 176 | > 177 | > - The created user can login remotely 178 | > - The container will error out if a password is not specified for the user 179 | > - No changes will be made if the user already exists 180 | > - Only a single user can be created at each launch 181 | 182 | ## Создание баз данных 183 | 184 | A new PostgreSQL database can be created by specifying the `DB_NAME` variable while starting the container. 185 | 186 | ```bash 187 | docker run --name postgresql -itd --restart always \ 188 | --env 'DB_NAME=dbname' \ 189 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 190 | ``` 191 | 192 | By default databases are created by copying the standard system database named `template1`. You can specify a different template for your database using the `DB_TEMPLATE` parameter. Refer to [Template Databases](http://www.postgresql.org/docs/9.4/static/manage-ag-templatedbs.html) for further information. 193 | 194 | Additionally, more than one database can be created by specifying a comma separated list of database names in `DB_NAME`. For example, the following command creates two new databases named `dbname1` and `dbname2`. 195 | 196 | ```bash 197 | docker run --name postgresql -itd --restart always \ 198 | --env 'DB_NAME=dbname1,dbname2' \ 199 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 200 | ``` 201 | 202 | ## Granting user access to a database 203 | 204 | If the `DB_USER` and `DB_PASS` variables are specified along with the `DB_NAME` variable, then the user specified in `DB_USER` will be granted access to all the databases listed in `DB_NAME`. Note that if the user and/or databases do not exist, they will be created. 205 | 206 | ```bash 207 | docker run --name postgresql -itd --restart always \ 208 | --env 'DB_USER=dbuser' --env 'DB_PASS=dbuserpass' \ 209 | --env 'DB_NAME=dbname1,dbname2' \ 210 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 211 | ``` 212 | 213 | In the above example `dbuser` with be granted access to both the `dbname1` and `dbname2` databases. 214 | 215 | # Enabling extensions 216 | 217 | The image also packages the [postgres contrib module](http://www.postgresql.org/docs/9.4/static/contrib.html). A comma separated list of modules can be specified using the `DB_EXTENSION` parameter. 218 | 219 | ```bash 220 | docker run --name postgresql -itd \ 221 | --env 'DB_NAME=db1,db2' --env 'DB_EXTENSION=unaccent,pg_trgm' \ 222 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 223 | ``` 224 | 225 | The above command enables the `unaccent` and `pg_trgm` modules on the databases listed in `DB_NAME`, namely `db1` and `db2`. 226 | 227 | > **NOTE**: 228 | > 229 | > This option deprecates the `DB_UNACCENT` parameter. 230 | 231 | ## Creating replication user 232 | 233 | Similar to the creation of a database user, a new PostgreSQL replication user can be created by specifying the `REPLICATION_USER` and `REPLICATION_PASS` variables while starting the container. 234 | 235 | ```bash 236 | docker run --name postgresql -itd --restart always \ 237 | --env 'REPLICATION_USER=repluser' --env 'REPLICATION_PASS=repluserpass' \ 238 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 239 | ``` 240 | 241 | > **Notes** 242 | > 243 | > - The created user can login remotely 244 | > - The container will error out if a password is not specified for the user 245 | > - No changes will be made if the user already exists 246 | > - Only a single user can be created at each launch 247 | 248 | *It is a good idea to create a replication user even if you are not going to use it as it will allow you to setup slave nodes and/or generate snapshots and backups when the need arises.* 249 | 250 | ## Setting up a replication cluster 251 | 252 | When the container is started, it is by default configured to act as a master node in a replication cluster. This means that you can scale your PostgreSQL database backend when the need arises without incurring any downtime. However do note that a replication user must exist on the master node for this to work. 253 | 254 | Begin by creating the master node of our cluster: 255 | 256 | ```bash 257 | docker run --name postgresql-master -itd --restart always \ 258 | --env 'DB_USER=dbuser' --env 'DB_PASS=dbuserpass' --env 'DB_NAME=dbname' \ 259 | --env 'REPLICATION_USER=repluser' --env 'REPLICATION_PASS=repluserpass' \ 260 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 261 | ``` 262 | 263 | Notice that no additional arguments are specified while starting the master node of the cluster. 264 | 265 | To create a replication slave the `REPLICATION_MODE` variable should be set to `slave` and additionally the `REPLICATION_HOST`, `REPLICATION_PORT`, `REPLICATION_SSLMODE`, `REPLICATION_USER` and `REPLICATION_PASS` variables should be specified. 266 | 267 | Create a slave node: 268 | 269 | ```bash 270 | docker run --name postgresql-slave01 -itd --restart always \ 271 | --link postgresql-master:master \ 272 | --env 'REPLICATION_MODE=slave' --env 'REPLICATION_SSLMODE=prefer' \ 273 | --env 'REPLICATION_HOST=master' --env 'REPLICATION_PORT=5432' \ 274 | --env 'REPLICATION_USER=repluser' --env 'REPLICATION_PASS=repluserpass' \ 275 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 276 | ``` 277 | 278 | *In the above command, we used docker links so that we can address the master node using the `master` alias in `REPLICATION_HOST`.* 279 | 280 | > **Note** 281 | > 282 | > - The default value of `REPLICATION_PORT` is `5432` 283 | > - The default value of `REPLICATION_SSLMODE` is `prefer` 284 | > - The value of `REPLICATION_USER` and `REPLICATION_PASS` should be the same as the ones specified on the master node. 285 | > - With [persistence](#persistence) in use, if the container is stopped and started, for the container continue to function as a slave you need to ensure that `REPLICATION_MODE=slave` is defined in the containers environment. In the absense of which the slave configuration will be turned off and the node will allow writing to it while having the last synced data from the master. 286 | 287 | And just like that with minimal effort you have a PostgreSQL replication cluster setup. You can create additional slaves to scale the cluster horizontally. 288 | 289 | Here are some important notes about a PostgreSQL replication cluster: 290 | 291 | - Writes can only occur on the master 292 | - Slaves are read-only 293 | - For best performance, limit the reads to the slave nodes 294 | 295 | ## Creating a snapshot 296 | 297 | Similar to a creating replication slave node, you can create a snapshot of the master by specifying `REPLICATION_MODE=snapshot`. 298 | 299 | Once the master node is created as specified in [Setting up a replication cluster](#setting-up-a-replication-cluster), you can create a snapshot using: 300 | 301 | ```bash 302 | docker run --name postgresql-snapshot -itd --restart always \ 303 | --link postgresql-master:master \ 304 | --env 'REPLICATION_MODE=snapshot' --env 'REPLICATION_SSLMODE=prefer' \ 305 | --env 'REPLICATION_HOST=master' --env 'REPLICATION_PORT=5432' \ 306 | --env 'REPLICATION_USER=repluser' --env 'REPLICATION_PASS=repluserpass' \ 307 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 308 | ``` 309 | 310 | The difference between a slave and a snapshot is that a slave is read-only and updated whenever the master data is updated (streaming replication), while a snapshot is read-write and is not updated after the initial snapshot of the data from the master. 311 | 312 | This is useful for developers to quickly snapshot the current state of a live database and use it for development/debugging purposes without altering the database on the live instance. 313 | 314 | ## Creating a backup 315 | 316 | Just as the case of setting up a slave node or generating a snapshot, you can also create a backup of the data on the master by specifying `REPLICATION_MODE=backup`. 317 | 318 | > The backups are generated with [pg_basebackup](http://www.postgresql.org/docs/9.4/static/app-pgbasebackup.html) using the replication protocol. 319 | 320 | Once the master node is created as specified in [Setting up a replication cluster](#setting-up-a-replication-cluster), you can create a point-in-time backup using: 321 | 322 | ```bash 323 | docker run --name postgresql-backup -it --rm \ 324 | --link postgresql-master:master \ 325 | --env 'REPLICATION_MODE=backup' --env 'REPLICATION_SSLMODE=prefer' \ 326 | --env 'REPLICATION_HOST=master' --env 'REPLICATION_PORT=5432' \ 327 | --env 'REPLICATION_USER=repluser' --env 'REPLICATION_PASS=repluserpass' \ 328 | --volume /srv/docker/backups/postgresql.$(date +%Y%m%d%H%M%S):/var/lib/postgresql \ 329 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 330 | ``` 331 | 332 | Once the backup is generated, the container will exit and the backup of the master data will be available at `/srv/docker/backups/postgresql.XXXXXXXXXXXX/`. Restoring the backup involves starting a container with the data in `/srv/docker/backups/postgresql.XXXXXXXXXXXX`. 333 | 334 | ## Command-line arguments 335 | 336 | You can customize the launch command of PostgreSQL server by specifying arguments for `postgres` on the `docker run` command. For example the following command enables connection logging: 337 | 338 | ```bash 339 | docker run --name postgresql -itd --restart always \ 340 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 -c log_connections=on 341 | ``` 342 | 343 | Please refer to the documentation of [postgres](http://www.postgresql.org/docs/9.6/static/app-postgres.html) for the complete list of available options. 344 | 345 | ## Журналы работы СУБД 346 | 347 | В образе PostgreSQL настроен на хранение лог файлов внутри выделенного тома `pg-log-master` смонтированного в каталог `/var/log/postgresql`. Логи записываются каждый час в новый файл, в имени файла при этом присутствует указание часа в течении которого в этот файл производилась запись. При наступлении новых суток содержимое файлов перезатираются новыми данными. Таким образом "рядом" с работающим сервером мы имеем журналы за последние 24 часа работы, что обеспечивает защиту от переполнения тома с файлами журналов. 348 | 349 | Для анализа журналов работы они передаются в режиме онлайн на хост с где запущен коллектор лог файлов (см. раздел Коллектор журналов) 350 | 351 | Для просомтра содержимого журнала работы вызвать `tail -f` внутри контейнера. Примерно так: 352 | 353 | ```bash 354 | docker exec -it postgresql tail -f /var/log/postgresql/postgresql-17.log 355 | ``` 356 | 357 | 358 | ## Коллектор журналов 359 | Для централизованного сбора и анализа журналов с помощью PgBadger сохраненные локальной на каждом сервере журналы работы СУБД отправляются в централизованное хранилище. Для этого с помощью сервиса `logstash` запускается контейнер и слушает порт 5000 для приема данных от службы `log-beats`. Входящие данные архивируются, сжимаются и складываются в файловую систему. 360 | 361 | Служба `log-beats` запускается рядом с запущенным контейнером СУБД подключаясь к тому с лог файлами СУБД обеспечивает их отправку в централизованное хранилище. 362 | 363 | 364 | Перед запуском необходимо в файле конфигурации filebeat/config/filebeat.yml указать ip адрес или днс имя хоста на котором запущен коллектор журналов. 365 | 366 | 367 | ## PgBadger 368 | 369 | Контейнер PgBadger собран с использованием сервиса `supervisord` который обеспечивает запуск нескольких служб внутри одного контейнера. Это веб сервер nginx для отображения отчетов сгенерированных из журналов работы и системный планировщик заданий `cron` который по расписанию запускает обработку журналов. По-умолчанию запуск задания анализа настроен на выполнение в пятую минуту каждого часа. 370 | 371 | После выполнения анализа проанализированный файл перемещается в поддиректорию (analyzed). 372 | 373 | Для просмотра статуса работы сервиса `supervisord` перейти в папку http://HOST-IP:9980/supervisor/ 374 | 375 | Логин/пароль: admin/admin 376 | 377 | Для принудительного запуска анализа записываемого журнала после старта контейнера выполнить команду: 378 | ``` 379 | docker exec pgbadger ls -l /var/log/pg-logs 380 | ``` 381 | Скопировать имя файла который необходимо проанализировать и запустить анализ указав имя файла: 382 | 383 | ``` 384 | docker exec pgbadger pgbadger -I -J 4 --start-monday -Z +03 \ 385 | ВЫБРАННЫЙ-ФАЙЛ.tar.gz \ 386 | -O /var/www/html/ 387 | ``` 388 | Для принудительного запуска скрипта анализа с переносом в архив выполнить: 389 | ``` 390 | docker exec pgbadger /bin/bash /usr/local/bin/start_analyze.sh 391 | ``` 392 | 393 | 394 | 395 | # UID/GID mapping 396 | 397 | The files and processes created by the container are owned by the `postgres` user that is internal to the container. In the absense of user namespace in docker the UID and GID of the containers `postgres` user may have different meaning on the host. 398 | 399 | For example, a user on the host with the same UID and/or GID as the `postgres` user of the container will be able to access the data in the persistent volumes mounted from the host as well as be able to KILL the `postgres` server process started by the container. 400 | 401 | To circumvent this issue you can specify the UID and GID for the `postgres` user of the container using the `USERMAP_UID` and `USERMAP_GID` variables respectively. 402 | 403 | For example, if you want to assign the `postgres` user of the container the UID and GID `999`: 404 | 405 | ```bash 406 | docker run --name postgresql -itd --restart always \ 407 | --env 'USERMAP_UID=999' --env 'USERMAP_GID=999' \ 408 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 409 | ``` 410 | # Параметры запуска контейнера 411 | 412 | Сгрупированы в файле [переменных окружения](runtime/env-defaults) 413 | 414 | > **Примечание** 415 | > каждая переменная имеет значение по умолчанию, например: 416 | > * `PG_TIMEZONE` - по умолчанию равно `Europe/Moscow`, то есть контейнер считает что он запускается в Московском времени 417 | 418 | # Обслуживание 419 | 420 | ## Обновление 421 | 422 | To upgrade to newer releases: 423 | 424 | 1. Download the updated Docker image: 425 | 426 | ```bash 427 | docker pull silverbulleters/ya-docker-postgresql-1c:9.6.5-5 428 | ``` 429 | 430 | 2. Stop the currently running image: 431 | 432 | ```bash 433 | docker stop postgresql 434 | ``` 435 | 436 | 3. Remove the stopped container 437 | 438 | ```bash 439 | docker rm -v postgresql 440 | ``` 441 | 442 | 4. Start the updated image 443 | 444 | ```bash 445 | docker run --name postgresql -itd \ 446 | [OPTIONS] \ 447 | silverbulleters/ya-docker-postgresql-1c:9.6.5-5 448 | ``` 449 | 450 | ## Shell Access 451 | 452 | For debugging and maintenance purposes you may want access the containers shell. If you are using Docker version `1.3.0` or higher you can access a running containers shell by starting `bash` using `docker exec`: 453 | 454 | ```bash 455 | docker exec -it postgresql bash 456 | ``` 457 | 458 | ### Перед стартом работы с 1С 459 | 460 | Если вы загрузили базу данных из DT файла в случае с 1С необходимо выполнить следующую последовательность команды 461 | 462 | ``` 463 | docker exec -it postgresql bash 464 | sudo su postgres 465 | vacuumdb -v -a -f -F -z 466 | ``` 467 | 468 | более подробно смотри 469 | 470 | * [на русском](https://postgrespro.ru/docs/postgrespro/current/app-vacuumdb) 471 | * [на английском](https://www.postgresql.org/docs/current/static/app-vacuumdb.html) 472 | 473 | # Базы данных, Тонкий тюннинг и прочее 474 | 475 | ## Когда можно использовать данный образ в продуктиве 476 | 477 | * Для GITLAB 478 | * Для SonarQube 479 | * Для 1С информационных систем работающих в режиме совместимости не ниже 8.3.9 480 | * образ использовался для баз данных размером `до 500 Gb` без использования сжатия на уровне файловых систем (только средствами TOAST) 481 | 482 | ## Ресурсы 483 | 484 | Обратите внимание контейнер использует для адаптации параметров запуска все выделенные ресурсы для Docker хоста, если вы хотите наложить ограничения на контейнер используейте параметры ограничений [Ограничения ресурсов](https://docs.docker.com/engine/admin/resource_constraints/) 485 | 486 | ## Сжатие TOAST 487 | 488 | Не все поля клиентское приложение может поместить в [TOAST](https://postgrespro.ru/docs/postgrespro/current/storage-toast), поэтому можно использовать скрипт для попытки сжатия в хранилище `EXTENDED` 489 | 490 | Скрипт использует oscript.io для реализации, для подключения используется системные переменные DBNAME 491 | 492 | Подробней в самом скрипте [./tools/compsess-status.os] 493 | 494 | ## Скрипты PLSQL 495 | 496 | * `postgres.public.make_tablespace(name, dir, owner)` - создает табличное простраство с идентификатором `name`, по адресу `dir` с владелцем owner - по умолчанию `postgres` 497 | 498 | ## Мониторинг 499 | 500 | в образ встроена возможность сбора журналов и статистики использования с помощью 501 | 502 | * `mamonsu` и https://github.com/zabbix/zabbix-docker 503 | * `pghero` 504 | * `powa` 505 | * `pgBadger` 506 | -------------------------------------------------------------------------------- /TODO-ru.md: -------------------------------------------------------------------------------- 1 | ## Список работ 2 | 3 | релиз "Предыдущий" 4 | 5 | * [X] сделать официальную сборку средствами hub.docker 6 | * [X] сделать сборку средствами travis (cirlce) 7 | * [X] проверить сборку и работу под Linux Docker 8 | * [X] локализовать README 9 | * [x] сделать экспериментальный скрипт по установке TOAST STORAGE `X` 10 | * [X] добавить мониторинг MamonSU 11 | * [X] дать возможность изменять TIMEZONE для PG из параметров docker 12 | * [X] добавить возможность upgrade песочниц для тестов 13 | 14 | релиз 6 15 | 16 | * [ ] сделать переменную окружения Zabbix-Server в docker-compose и задокументировать 17 | * [ ] подключить PGBadger демон и настроить log-collector 18 | * [X] добавить возможность работы с tmpfs в части временных таблиц PG 19 | * [ ] починить pghero 20 | 21 | релиз 7 22 | 23 | * [ ] сделать автотюнинг на LOCK.WAL 24 | * [ ] сделать Docker-Compose для работы с Consul.io как показать примера DNS сервисов и нескольких серверов PG на одном хосте 25 | * [ ] добавить автотесты средствами pgbench с параметрами ожидаемой производительности в docker-compose 26 | * [ ] отмигрироваться на наш базовый образ Ubuntu 27 | * [ ] вставить презентационный скрипт автоматического HA средствами repmanager 28 | * [ ] добавить merge-strategies для для работы с upstream 29 | * [ ] отмигрировать настройки barman 30 | * [ ] вставить шардирование и секционирование от PostgreSQL.pro 31 | * [ ] проверить на безопасность образ в части CERT 32 | * [ ] перевести весь README.md 33 | * [ ] включить поддержку Docker-Macnine для реализации master-slave хостов 34 | 35 | релиз 8 36 | 37 | * [ ] аудировать и перенести наработки с курса по PG, pgSteroids и pgClass, pgDock 38 | * [ ] проверить концепции pgPool, pgBounser 39 | * [ ] реализовать автоматический failover (stolon, pgrewind, repmanager) без потери сессии 1С - вомзожмно с миграцией IP 40 | * [ ] возможно только с логической репликацией которая лучше сделана в 10.x PG 41 | * [ ] заменить в GitLab и SonarQube образ на `silverbulleters/ya-docker` 42 | * [ ] изучить форки на предмет интересной функциональности 43 | * [ ] часть функций реализовать на oscript - особенно логику адаптационных postgresql.conf, хотя пока можено олстановиться на `mamonsu tune` 44 | 45 | релиз SUPER 46 | 47 | * [ ] incorporate alien tech -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 9.6-5.5 2 | -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | Vagrant.configure(2) do |config| 5 | config.vm.box = "bessonovevgen/ubuntu-server-16" 6 | 7 | @provider = "virtualbox" 8 | 9 | # config.vm.provider "hyperv" do |v, override| 10 | # override.vm.box = "kmm/ubuntu-xenial64" 11 | # @provider = "hyperv" 12 | # end 13 | 14 | # @dockeruser = if (@provider == "virtualbox") ; then "ubuntu" else "vagrant" end 15 | @dockeruser = "vagrant" 16 | 17 | config.vm.box_check_update = false 18 | # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies 19 | # such as FTP and Heroku are also available. See the documentation at 20 | # https://docs.vagrantup.com/v2/push/atlas.html for more information. 21 | # config.push.define "atlas" do |push| 22 | # push.app = "vanessa-dockers/pgsteroids" 23 | # end 24 | config.vm.define "docker" do |docker| 25 | docker.vm.hostname = "docker" 26 | docker.vm.network "forwarded_port", guest: 5432, host: 5432 27 | docker.vm.network "forwarded_port", guest: 8081, host: 8081 28 | docker.vm.network "forwarded_port", guest: 8888, host: 8888 29 | docker.vm.network "forwarded_port", guest: 9997, host: 9997 30 | docker.vm.network "forwarded_port", guest: 9998, host: 9998 31 | docker.vm.network "forwarded_port", guest: 9999, host: 9999 32 | docker.vm.provider "virtualbox" do |vb| 33 | vb.memory = "4096" 34 | vb.name = "docker" 35 | end 36 | 37 | docker.vm.provider "hyperv" do |hv| 38 | hv.vmname = "docker" 39 | end 40 | # run the provisioning only is the first 'vagrant up' 41 | # config.vm.provision :shell, :inline = @ 42 | if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/first/" + @provider + "/id").empty? 43 | # Install Docker 44 | pkg_cmd = "curl -sSL https://get.docker.com/ | sh;" 45 | # Add user to the docker group and install packadges 46 | pkg_cmd << "usermod -a -G docker " + @dockeruser + " ; " 47 | pkg_cmd << "apt-get update -y -q; " 48 | pkg_cmd << "apt-get install dnsmasq python3-pip python-psycopg2 libdbd-pg-perl libdbi-perl docker-compose mc -y -q; " 49 | config.vm.provision :shell, :inline => pkg_cmd 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /change-size-vm-disk.md: -------------------------------------------------------------------------------- 1 | # Изменение размера диска виртуальной машины 2 | 3 | ## docker for windows (hyperv) 4 | TODO 5 | 6 | 7 | 8 | ## docker toolbox (virtualbox) 9 | Необходимо закрыть kitematic если запущен. 10 | 11 | В консоли Docker Quickstart Terminal выполним команду удаления машины 12 | ``` 13 | $ docker-machine rm default 14 | ``` 15 | 16 | И пересоздадим её с размером диска 100 Гб. 17 | ``` 18 | $ docker-machine create -d virtualbox --virtualbox-disk-size "102400" default 19 | ``` 20 | 21 | Для того чтобы всегда создавалась виртулаьная машина с необходимым размером диска добавить в скрипт запуска `start.sh` (%ProgramFiles%\Docker Toolbox\) на шаге `Checking if machine $VM exists` 22 | 23 | в cтроке 24 | ``` 25 | "${DOCKER_MACHINE}" create -d virtualbox $PROXY_ENV "${VM}" 26 | ``` 27 | параметр `--virtualbox-disk-size "102400"` 28 | 29 | Чтобы получилась такая строка 30 | ``` 31 | "${DOCKER_MACHINE}" create -d virtualbox --virtualbox-disk-size "102400" $PROXY_ENV "${VM}" 32 | ``` 33 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | services: 3 | - docker 4 | 5 | dependencies: 6 | override: 7 | - docker info 8 | 9 | test: 10 | override: 11 | - docker build -t silverbulleters/ya-docker-postgresql-1c . 12 | - docker build -t silverbulleters/pgbadger pgbadger/ 13 | - docker run -d --name=postgresql silverbulleters/ya-docker-postgresql-1c; sleep 10 14 | - docker run -it --volumes-from=postgresql silverbulleters/ya-docker-postgresql-1c sudo -u postgres -H psql -c "\conninfo" 15 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | PostgreSQL: 4 | restart: always 5 | build: 6 | context: . 7 | image: silverbulleters/ya-docker-postgresql-1c:latest 8 | # tmpfs: 9 | # - /var/lib/postgresql/temptblspc 10 | ports: 11 | - "5432:5432" 12 | environment: 13 | - USETMPFS_FOR_TMP=true 14 | - DEBUG=true 15 | - PG_PASSWORD=somepass 16 | - DB_USER=onecuser 17 | - DB_PASS=anotherpass 18 | - DB_NAME=powa,mamonsu 19 | - POWA_DATABASE=powa 20 | - DB_TEMPLATE= 21 | - DB_EXTENSION= 22 | - REPLICATION_MODE= 23 | - REPLICATION_USER=repluser 24 | - REPLICATION_PASS=replpass 25 | - REPLICATION_SSLMODE= 26 | - PG_TUNE_BY_MAMONSU=true 27 | - PG_MAX_CONNECTION=130 28 | - PG_TIMEZONE=Europe/Moscow 29 | volumes: 30 | - pg-data-master:/var/lib/postgresql/9.6/main 31 | - pg-temptbl-master:/var/lib/postgresql/temptblspc 32 | - pg-v81cdata-master:/var/lib/postgresql/v81c_data 33 | - pg-v81cindex-master:/var/lib/postgresql/v81c_index 34 | - pg-log-master:/var/log/postgresql 35 | 36 | 37 | PostgreSQLSlave: 38 | image: silverbulleters/ya-docker-postgresql-1c:latest 39 | # tmpfs: 40 | # - /var/lib/postgresql/temptblspc 41 | links: 42 | - PostgreSQL:master 43 | environment: 44 | - USETMPFS-FOR-TMP=true 45 | - REPLICATION_MODE=slave 46 | - REPLICATION_USER=repluser 47 | - REPLICATION_PASS=replpass 48 | - REPLICATION_HOST=master 49 | - REPLICATION_PORT=5432 50 | - REPLICATION_SSLMODE=prefer 51 | volumes: 52 | - pg-data-slave:/var/lib/postgresql/9.6/main 53 | - pg-temptbl-slave:/var/lib/postgresql/temptblspc 54 | - pg-v81cdata-slave:/var/lib/postgresql/v81c_data 55 | - pg-v81cindex-slave:/var/lib/postgresql/v81c_index 56 | - pg-log-slave:/var/log/postgresql 57 | 58 | PgAdminDBA: 59 | image: dpage/pgadmin4 60 | ports: 61 | - 8880:80 62 | environment: 63 | - PGADMIN_DEFAULT_EMAIL=team@silverbulleters.org 64 | - PGADMIN_DEFAULT_PASSWORD=somepass 65 | # - PGADMIN_ENABLE_TLS= 66 | # - PGADMIN_SERVER_NAME= 67 | volumes: 68 | - pgadmin-data:/var/lib/pgadmin/ 69 | links: 70 | - PostgreSQL:master 71 | - PostgreSQLSlave:slave 72 | 73 | PowaWeb: 74 | build: 75 | context: powa-web 76 | image: silverbulleters/powa-web:latest 77 | ports: 78 | - 8899:8888 79 | links: 80 | - PostgreSQL:master 81 | - PostgreSQLSlave:slave 82 | 83 | pghero-db1: 84 | image: ankane/pghero:v2.0.2 85 | environment: 86 | - DATABASE_URL=postgres://postgres:somepass@master-db:5432/frag-20 87 | links: 88 | - PostgreSQL:master-db 89 | ports: 90 | - 10000:8080 91 | 92 | pgbadger: 93 | container_name: pgbadger 94 | build: 95 | context: pgbadger 96 | image: silverbulleters/pgbadger:latest 97 | ports: 98 | - 9980:80 99 | volumes: 100 | - logstash-data:/var/log/pg-logs 101 | # - ./:/srv 102 | 103 | mamonsu: 104 | container_name: mamonsu 105 | build: 106 | context: mamonsu 107 | image: silverbulleters/mamonsu:latest 108 | environment: 109 | - ZABBIX_URL=http://127.0.0.1:8081/ 110 | - ZABBIX_SERVER=127.0.0.1 111 | - ZABBIX_USER=Admin 112 | - ZABBIX_PASSWD=zabbix 113 | - PG_SERVER=127.0.0.1 114 | - PG_USER=postgres 115 | - MAMONSU_DB=mamonsu 116 | ports: 117 | - 10050:10050 118 | links: 119 | - PostgreSQL:master 120 | 121 | log-beats: 122 | build: 123 | context: filebeat/ 124 | image: silverbulleters/filebeat:6.1.0 125 | volumes: 126 | - pg-log-master:/var/log/postgresql:ro 127 | 128 | logstash: 129 | build: 130 | context: logstash/ 131 | image: silverbulleters/logstash:6.1.0 132 | volumes: 133 | - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro 134 | - ./logstash/pipeline:/usr/share/logstash/pipeline:ro 135 | - logstash-data:/var/log/pg-logs/ 136 | ports: 137 | - "5000:5000" 138 | - "5044:5044" 139 | - "9600:9600" 140 | environment: 141 | LS_JAVA_OPTS: "-Xmx256m -Xms256m" 142 | 143 | 144 | volumes: 145 | pg-data-master: 146 | pg-temptbl-master: 147 | pg-v81cdata-master: 148 | pg-v81cindex-master: 149 | pg-log-master: 150 | 151 | pg-data-slave: 152 | pg-temptbl-slave: 153 | pg-v81cdata-slave: 154 | pg-v81cindex-slave: 155 | pg-log-slave: 156 | 157 | pgadmin-data: 158 | 159 | logstash-data: 160 | -------------------------------------------------------------------------------- /entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | source ${PG_APP_HOME}/functions 4 | 5 | [[ ${DEBUG} == true ]] && set -x 6 | 7 | # allow arguments to be passed to postgres 8 | if [[ ${1:0:1} = '-' ]]; then 9 | EXTRA_ARGS="$@" 10 | set -- 11 | elif [[ ${1} == postgres || ${1} == $(which postgres) ]]; then 12 | EXTRA_ARGS="${@:2}" 13 | set -- 14 | fi 15 | 16 | # default behaviour is to launch postgres 17 | if [[ -z ${1} ]]; then 18 | map_uidgid 19 | 20 | create_datadir 21 | create_certdir 22 | create_logdir 23 | create_rundir 24 | 25 | set_resolvconf_perms 26 | 27 | configure_postgresql 28 | 29 | echo "Starting PostgreSQL ${PG_VERSION}..." 30 | exec start-stop-daemon --start --chuid ${PG_USER}:${PG_USER} \ 31 | --exec ${PG_BINDIR}/postgres -- -D ${PG_DATADIR} ${EXTRA_ARGS} 32 | else 33 | exec "$@" 34 | fi 35 | 36 | -------------------------------------------------------------------------------- /filebeat/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/beats/filebeat:6.1.0 2 | 3 | COPY config /usr/share/filebeat/ 4 | 5 | USER root 6 | 7 | RUN chown -R filebeat /usr/share/filebeat/* 8 | RUN chmod -R go-w /usr/share/filebeat/* 9 | 10 | USER filebeat 11 | 12 | ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] 13 | CMD ["-e"] 14 | -------------------------------------------------------------------------------- /filebeat/config/filebeat.yml: -------------------------------------------------------------------------------- 1 | filebeat.config: 2 | prospectors: 3 | path: /usr/share/filebeat/prospectors.d/*.yml 4 | reload.enabled: false 5 | modules: 6 | path: /usr/share/filebeat/modules.d/*.yml 7 | reload.enabled: false 8 | 9 | processors: 10 | - add_cloud_metadata: 11 | 12 | output.logstash: 13 | hosts: ["127.0.0.1:5000"] 14 | -------------------------------------------------------------------------------- /filebeat/config/prospectors.d/default.yml: -------------------------------------------------------------------------------- 1 | - input_type: log 2 | paths: 3 | - /var/log/postgresql/postgresql-*.log 4 | multiline.pattern: '^[[:space:]]' 5 | multiline.negate: false 6 | multiline.match: after 7 | 8 | 9 | # - input_type: log 10 | # paths: 11 | # - /val/log/*.log 12 | -------------------------------------------------------------------------------- /logstash/Dockerfile: -------------------------------------------------------------------------------- 1 | # https://github.com/elastic/logstash-docker 2 | FROM docker.elastic.co/logstash/logstash-oss:6.1.0 3 | 4 | USER root 5 | 6 | RUN mkdir /var/log/pg-logs/ 7 | 8 | RUN chown -R logstash:logstash /var/log/pg-logs/ 9 | 10 | USER logstash 11 | 12 | EXPOSE 9600 5044 5000 13 | 14 | ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] 15 | 16 | 17 | # Add your logstash plugins setup here 18 | # Example: RUN logstash-plugin install logstash-filter-json 19 | 20 | 21 | -------------------------------------------------------------------------------- /logstash/config/logstash.yml: -------------------------------------------------------------------------------- 1 | --- 2 | ## Default Logstash configuration from logstash-docker. 3 | ## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml 4 | # 5 | http.host: "0.0.0.0" 6 | path.config: /usr/share/logstash/pipeline 7 | -------------------------------------------------------------------------------- /logstash/pipeline/logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | beats { 3 | port => 5000 4 | } 5 | } 6 | 7 | ## Add your filters / logstash plugins configuration here 8 | 9 | # output { 10 | # elasticsearch { 11 | # hosts => "elasticsearch:9200" 12 | # } 13 | # } 14 | # output { 15 | # stdout {} 16 | # } 17 | 18 | output { 19 | file { 20 | path => "/var/log/pg-logs/%{+YYYY-MM-dd-H}-postgresql.log.gz" 21 | codec => 22 | line { 23 | format => "%{message}" 24 | } 25 | gzip => true 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /mamonsu/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3 2 | 3 | RUN pip install mamonsu 4 | 5 | ENV ZABBIX_URL=http://127.0.0.1:8081/ \ 6 | ZABBIX_USER=Admin \ 7 | ZABBIX_PASSWD=zabbix \ 8 | ZABBIX_SERVER=127.0.0.1 \ 9 | PG_SERVER=127.0.0.1 \ 10 | PG_USER=postgres \ 11 | MAMONSU_DB=mamonsu 12 | 13 | ADD agent.conf /etc/mamonsu/ 14 | 15 | COPY mamonsu-entrypoint.sh / 16 | 17 | COPY init-start.sh / 18 | 19 | # ENTRYPOINT ["/mamonsu-entrypoint.sh"] 20 | ENTRYPOINT ["/bin/bash"] 21 | CMD ["/mamonsu-entrypoint.sh"] 22 | 23 | EXPOSE 10050 24 | 25 | # CMD ["mamonsu -c /etc/mamonsu/agent.conf -p /var/run/mamonsu.pid"] 26 | # CMD ["mamonsu"] 27 | # ENTRYPOINT ["mamonsu"] 28 | 29 | # CMD ["-c /etc/mamonsu/agent.conf"] -------------------------------------------------------------------------------- /mamonsu/README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | Начальный запуск mamonsu. 4 | Необходимо указать IP-адрес сервера, за которым будете наблюдать и имя базы данных, имя и пароль пользователя Postgres, используя которые mamonsu будет подключаться к серверу, адрес и учетные данные для доступа к web интерфейсу Zabbix сервера. 5 | 6 | ``` 7 | docker run --rm \ 8 | -e ZABBIX_URL=http://192.168.99.100:8081/ \ 9 | -e PG_SERVER=192.168.99.100 \ 10 | -e PG_USER=postgres \ 11 | -e MAMONSU_DB=mamonsu \ 12 | silverbulleters/mamonsu init-start.sh 13 | ``` 14 | 15 | В результате mamonsu создаёт в указанной базе данных несколько таблиц с префиксом mamonsu. Не удаляйте эти таблицы, так как они нужны для работы mamonsu. А также производит экспорт шаблона мониторинга на сервер Zabbix. 16 | 17 | 18 | WIP Создание группы и хоста 19 | TODO - получение id элементов или задание их принудительно 20 | 21 | docker run -it --entrypoint "mamonsu" silverbulleters/mamonsu zabbix hostgroup create pg --url=http://192.168.99.100:8081/ --user=Admin --password=zabbix 22 | 23 | 24 | docker run -it --entrypoint "mamonsu" silverbulleters/mamonsu zabbix host create pg-master 15 10254 192.168.99.100 --url=http://192.168.99.100:8081/ --user=Admin --password=zabbix 25 | 26 | где 27 | 15 - id группы хостов созданной на предыдущем шаге 28 | 10254 - id загруженного шаблона 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /mamonsu/agent.conf: -------------------------------------------------------------------------------- 1 | [postgres] 2 | enabled = True 3 | user = postgres 4 | password = somepass 5 | database = mamonsu 6 | host = master 7 | port = 5432 8 | application_name = mamonsu 9 | query_timeout = 10 10 | 11 | [system] 12 | enabled = True 13 | 14 | [sender] 15 | queue = 2048 16 | 17 | [agent] 18 | enabled = True 19 | host = 0.0.0.0 20 | port = 10050 21 | 22 | [plugins] 23 | enabled = False 24 | directory = /etc/mamonsu/plugins 25 | 26 | [zabbix] 27 | enabled = True 28 | client = pg-master 29 | address = 127.0.0.1 30 | port = 10051 31 | 32 | [metric_log] 33 | enabled = False 34 | directory = /var/log/mamonsu 35 | max_size_mb = 1024 36 | 37 | [log] 38 | file = None 39 | level = INFO 40 | format = [%(levelname)s] %(asctime)s - %(name)s - %(message)s 41 | 42 | [health] 43 | max_memory_usage = 41943040 44 | 45 | [databases] 46 | min_rows = 50 47 | bloat_scale = 0.2 48 | 49 | [pghealth] 50 | uptime = 600 51 | cache = 80 52 | 53 | [xlog] 54 | lag_more_then_in_sec = 300 55 | 56 | [checkpoint] 57 | max_checkpoint_by_wal_in_hour = 12 58 | 59 | [oldest] 60 | max_xid_age = 18000000 61 | max_query_time = 18000 62 | 63 | [cfs] 64 | force_enable = False 65 | 66 | [disksizes] 67 | vfs_percent_free = 10 68 | vfs_inode_percent_free = 10 69 | 70 | [systemuptime] 71 | uptime = 300 72 | 73 | -------------------------------------------------------------------------------- /mamonsu/init-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | mamonsu bootstrap \ 3 | --host ${PG_SERVER} \ 4 | --dbname=${MAMONSU_DB} \ 5 | --username=${PG_USER} -M ${PG_USER} 6 | 7 | mamonsu export template template.xml 8 | 9 | mamonsu zabbix template export template.xml \ 10 | --url=${ZABBIX_URL} \ 11 | --user=${ZABBIX_USER} \ 12 | --password=${ZABBIX_PASSWD} 13 | -------------------------------------------------------------------------------- /mamonsu/mamonsu-entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sed -i "s|address = .*|address = ${ZABBIX_SERVER}|" /etc/mamonsu/agent.conf 4 | 5 | exec mamonsu -c /etc/mamonsu/agent.conf -p /var/run/mamonsu.pid -------------------------------------------------------------------------------- /pgbadger/Dockerfile: -------------------------------------------------------------------------------- 1 | # Pull from the ubuntu:14.04 image 2 | FROM ubuntu:14.04 3 | 4 | # Set the author 5 | MAINTAINER team@silverbulleters.org 6 | 7 | ENV DEBIAN_FRONTEND noninteractive 8 | 9 | # Update cache and install packages 10 | RUN apt-get update && apt-get -y install \ 11 | software-properties-common \ 12 | # python-software-properties \ 13 | debian-archive-keyring \ 14 | wget \ 15 | curl \ 16 | make \ 17 | cron \ 18 | supervisor \ 19 | rsyslog 20 | 21 | # Download Nginx signing key 22 | RUN apt-key adv --recv-keys --keyserver keyserver.ubuntu.com C300EE8C 23 | 24 | # Add to repository sources list 25 | RUN add-apt-repository ppa:nginx/stable 26 | 27 | # Update cache and install Nginx 28 | RUN apt-get update && apt-get -y install \ 29 | nginx 30 | 31 | RUN apt-get clean; \ 32 | rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 33 | 34 | RUN mkdir -p /var/log/supervisor 35 | 36 | RUN sed -i 's/#cron/cron/g' /etc/rsyslog.d/50-default.conf 37 | 38 | # Turn off daemon mode 39 | # Reference: http://stackoverflow.com/questions/18861300/how-to-run-nginx-within-docker-container-without-halting 40 | RUN echo "\ndaemon off;" >> /etc/nginx/nginx.conf 41 | 42 | # Run nginx and php-fpm as root 43 | RUN sed -i 's/www-data/root/g' /etc/nginx/nginx.conf 44 | 45 | # Backup the default configurations 46 | RUN mv /etc/nginx/sites-available/default /etc/nginx/sites-available/default.original 47 | 48 | # Copy nginx site conf 49 | COPY nginx.conf /etc/nginx/sites-available/default 50 | 51 | # Remove default page 52 | RUN rm /var/www/html/index.nginx-debian.html 53 | 54 | # Copy the supervisord.conf file 55 | COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf 56 | 57 | # Add pgbadger 58 | ENV PGBGRVER=9.2 59 | 60 | ADD https://github.com/dalibo/pgbadger/archive/v${PGBGRVER}.tar.gz /tmp/ 61 | 62 | WORKDIR /tmp/ 63 | 64 | RUN tar xzf v${PGBGRVER}.tar.gz && \ 65 | rm v${PGBGRVER}.tar.gz 66 | 67 | WORKDIR /tmp/pgbadger-${PGBGRVER} 68 | 69 | RUN perl Makefile.PL && make && make install 70 | 71 | RUN pgbadger --version 72 | 73 | COPY start_analyze.sh /usr/local/bin/start_analyze.sh 74 | 75 | # Copy the crontab file 76 | COPY start-pgbadger-cron /etc/cron.d/start-pgbadger-cron 77 | RUN chmod 0644 /etc/cron.d/start-pgbadger-cron 78 | 79 | # Mount volumes 80 | VOLUME ["/etc/nginx/certs", "/etc/nginx/conf.d", "/var/www/html"] 81 | 82 | # Boot up supervisord when container is started 83 | CMD ["/usr/bin/supervisord"] 84 | 85 | # Set the current working directory 86 | WORKDIR /var/www/html 87 | 88 | EXPOSE 80 89 | -------------------------------------------------------------------------------- /pgbadger/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | index index.php index.html; 3 | server_name _; 4 | root /var/www/html; 5 | 6 | location /supervisor/ { 7 | proxy_pass http://unix:/var/run/supervisord.sock:/; 8 | proxy_set_header Host $host; 9 | proxy_set_header X-Real-IP $remote_addr; 10 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 11 | proxy_read_timeout 600; 12 | include proxy_params; 13 | } 14 | 15 | location = /favicon.ico { 16 | log_not_found off; 17 | access_log off; 18 | } 19 | location = /robots.txt { 20 | allow all; 21 | log_not_found off; 22 | access_log off; 23 | } 24 | 25 | # Deny hidden files 26 | location ^~ /. { 27 | deny all; 28 | } 29 | 30 | location ~ \.php$ { 31 | try_files $uri =404; 32 | include fastcgi_params; 33 | fastcgi_split_path_info ^(.+\.php)(/.+)$; 34 | fastcgi_index index.php; 35 | fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; 36 | fastcgi_intercept_errors on; 37 | fastcgi_ignore_client_abort on; 38 | fastcgi_connect_timeout 60; 39 | fastcgi_send_timeout 180; 40 | fastcgi_read_timeout 180; 41 | fastcgi_buffer_size 128k; 42 | fastcgi_buffers 256 16k; 43 | fastcgi_busy_buffers_size 256k; 44 | fastcgi_temp_file_write_size 256k; 45 | fastcgi_max_temp_file_size 0; 46 | fastcgi_pass unix:/var/run/php5-fpm.sock; 47 | } 48 | 49 | location ~* \.(xml|ogg|ogv|svg|svgz|eot|otf|woff|mp4|ttf|css|rss|atom|js|jpg|jpeg|gif|png|ico|zip|tgz|gz|rar|bz2|doc|xls|exe|ppt|tar|mid|midi|wav|bmp|rtf)\$ { 50 | expires max; 51 | add_header Pragma "public"; 52 | add_header Cache-Control "public, must-revalidate, proxy-revalidate"; 53 | access_log off; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pgbadger/start-pgbadger-cron: -------------------------------------------------------------------------------- 1 | SHELL=/bin/sh 2 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 3 | 4 | 5 * * * * root start_analyze.sh 5 | -------------------------------------------------------------------------------- /pgbadger/start_analyze.sh: -------------------------------------------------------------------------------- 1 | REPORT_DIR=/var/www/html/ 2 | PARALLEL_PROC=4 3 | PGBADGER_TZ=+03 4 | LOG_DIR=/var/log/pg-logs 5 | PARCED_LOG_DIR=${LOG_DIR}/analyzed 6 | 7 | cd ${LOG_DIR} 8 | 9 | if [[ ! -d ${REPORT_DIR} ]]; then 10 | echo create report dir in ${REPORT_DIR} 11 | mkdir -p ${REPORT_DIR} 12 | else 13 | echo ${REPORT_DIR} already exist 14 | fi 15 | 16 | if [[ ! -d ${PARCED_LOG_DIR} ]]; then 17 | echo create archive log dir in ${PARCED_LOG_DIR} 18 | mkdir -p ${PARCED_LOG_DIR} 19 | else 20 | echo ${PARCED_LOG_DIR} already exist 21 | fi 22 | 23 | 24 | file=`ls -1rt *.log.gz | head -1` 25 | pgbadger -I -q -J ${PARALLEL_PROC} --start-monday -Z ${PGBADGER_TZ} $file \ 26 | -O ${REPORT_DIR} && \ 27 | mv $file ${PARCED_LOG_DIR}/$file 28 | -------------------------------------------------------------------------------- /pgbadger/supervisord.conf: -------------------------------------------------------------------------------- 1 | [supervisord] 2 | nodaemon=true 3 | 4 | [unix_http_server] 5 | file=/var/run/supervisord.sock 6 | chmod=0777 7 | chown=nobody:nogroup 8 | username=admin 9 | password=admin 10 | 11 | [program:nginx] 12 | command=/usr/sbin/nginx 13 | 14 | [program:cron] 15 | command=/usr/sbin/cron -f 16 | 17 | [program:rsyslog] 18 | command=/usr/sbin/rsyslogd -n 19 | -------------------------------------------------------------------------------- /powa-web/Dockerfile: -------------------------------------------------------------------------------- 1 | #vim:set ft=dockerfile 2 | FROM debian:jessie 3 | LABEL maintainer="Julien Rouhaud " 4 | 5 | WORKDIR /usr/local/src 6 | 7 | RUN nc -z w3 consul-io.node.cloud.consul 3142 && echo 'Acquire::http { Proxy "http://consul-io.node.cloud.consul:3142"; };' >> /etc/apt/apt.conf.d/01proxy || echo "not found proxy, skip" 8 | RUN echo 'precedence ::ffff:0:0/96 100' >> /etc/gai.conf 9 | 10 | RUN apt-get update && apt-get install -y \ 11 | libpq5 \ 12 | libpq-dev \ 13 | python \ 14 | python-dev \ 15 | python-pip \ 16 | && pip install powa-web \ 17 | && apt-get purge -y --auto-remove libpq-dev python-dev python-pip \ 18 | && rm -rf /var/lib/apt/lists/* 19 | 20 | COPY powa-web.conf /etc/ 21 | 22 | EXPOSE 8888 23 | CMD ["powa-web"] 24 | -------------------------------------------------------------------------------- /powa-web/powa-web.conf: -------------------------------------------------------------------------------- 1 | servers={ 2 | 'PostgreSQLMaster': { 3 | 'host': 'master', 4 | 'port': '5432', 5 | 'database': 'powa' 6 | }, 7 | 'PostgreSQLSlave': { 8 | 'host': 'slave', 9 | 'port': '5432', 10 | 'database': 'powa' 11 | } 12 | } 13 | 14 | cookie_secret="very strong secrete phrase" 15 | -------------------------------------------------------------------------------- /psql.bat: -------------------------------------------------------------------------------- 1 | 2 | docker exec -it yadockerpostgresql1c_PostgreSQL_1 su postgres -c psql %* 3 | -------------------------------------------------------------------------------- /psql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker exec -it yadockerpostgresql1c_PostgreSQL_1 su postgres -c psql $@ 4 | -------------------------------------------------------------------------------- /runtime/env-defaults: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | PG_SSL=${PG_SSL:-} 4 | 5 | PG_TRUST_LOCALNET=${PG_TRUST_LOCALNET:-$PSQL_TRUST_LOCALNET} # backward compatibility 6 | PG_TRUST_LOCALNET=${PG_TRUST_LOCALNET:-false} 7 | 8 | REPLICATION_MODE=${REPLICATION_MODE:-$PSQL_MODE} # backward compatibility 9 | REPLICATION_MODE=${REPLICATION_MODE:-} 10 | REPLICATION_USER=${REPLICATION_USER:-} 11 | REPLICATION_PASS=${REPLICATION_PASS:-} 12 | REPLICATION_HOST=${REPLICATION_HOST:-} 13 | REPLICATION_PORT=${REPLICATION_PORT:-5432} 14 | REPLICATION_SSLMODE=${REPLICATION_SSLMODE:-prefer} 15 | 16 | DB_NAME=${DB_NAME:-} 17 | DB_USER=${DB_USER:-} 18 | DB_PASS=${DB_PASS:-} 19 | DB_TEMPLATE=${DB_TEMPLATE:-template1} 20 | 21 | DB_EXTENSION=${DB_EXTENSION:-} 22 | 23 | POWA_DATABASE=${POWA_DATABASE:-powa} 24 | 25 | PG_MAX_CONNECTION=${PG_MAX_CONNECTION:-100} 26 | PG_TUNE_BY_MAMONSU=${PG_TUNE_BY_MAMONSU:-true} 27 | 28 | PG_TIMEZONE=${PG_TIMEZONE:-Europe/Moscow} 29 | 30 | USETMPFS_FOR_TMP=${USETMPFS_FOR_TMP:-true} 31 | -------------------------------------------------------------------------------- /runtime/functions: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | source ${PG_APP_HOME}/env-defaults 4 | 5 | PG_CONF=${PG_DATADIR}/postgresql.conf 6 | PG_HBA_CONF=${PG_DATADIR}/pg_hba.conf 7 | PG_IDENT_CONF=${PG_DATADIR}/pg_ident.conf 8 | PG_RECOVERY_CONF=${PG_DATADIR}/recovery.conf 9 | 10 | ## Execute command as PG_USER 11 | exec_as_postgres() { 12 | sudo -HEu ${PG_USER} "$@" 13 | } 14 | 15 | map_uidgid() { 16 | USERMAP_ORIG_UID=$(id -u ${PG_USER}) 17 | USERMAP_ORIG_GID=$(id -g ${PG_USER}) 18 | USERMAP_GID=${USERMAP_GID:-${USERMAP_UID:-$USERMAP_ORIG_GID}} 19 | USERMAP_UID=${USERMAP_UID:-$USERMAP_ORIG_UID} 20 | if [[ ${USERMAP_UID} != ${USERMAP_ORIG_UID} ]] || [[ ${USERMAP_GID} != ${USERMAP_ORIG_GID} ]]; then 21 | echo "Adapting uid and gid for ${PG_USER}:${PG_USER} to $USERMAP_UID:$USERMAP_GID" 22 | groupmod -o -g ${USERMAP_GID} ${PG_USER} 23 | sed -i -e "s|:${USERMAP_ORIG_UID}:${USERMAP_GID}:|:${USERMAP_UID}:${USERMAP_GID}:|" /etc/passwd 24 | fi 25 | } 26 | 27 | create_datadir() { 28 | echo "Initializing datadir..." 29 | mkdir -p ${PG_HOME} 30 | if [[ -d ${PG_DATADIR} ]]; then 31 | find ${PG_DATADIR} -type f -exec chmod 0600 {} \; 32 | find ${PG_DATADIR} -type d -exec chmod 0700 {} \; 33 | fi 34 | chown -R ${PG_USER}:${PG_USER} ${PG_HOME} 35 | } 36 | 37 | create_certdir() { 38 | echo "Initializing certdir..." 39 | mkdir -p ${PG_CERTDIR} 40 | [[ -f ${PG_CERTDIR}/server.crt ]] && chmod 0644 ${PG_CERTDIR}/server.crt 41 | [[ -f ${PG_CERTDIR}/server.key ]] && chmod 0640 ${PG_CERTDIR}/server.key 42 | chmod 0755 ${PG_CERTDIR} 43 | chown -R root:${PG_USER} ${PG_CERTDIR} 44 | } 45 | 46 | create_logdir() { 47 | echo "Initializing logdir..." 48 | mkdir -p ${PG_LOGDIR} 49 | chmod -R 1775 ${PG_LOGDIR} 50 | chown -R root:${PG_USER} ${PG_LOGDIR} 51 | } 52 | 53 | create_rundir() { 54 | echo "Initializing rundir..." 55 | mkdir -p ${PG_RUNDIR} ${PG_RUNDIR}/${PG_VERSION}-main.pg_stat_tmp 56 | chmod -R 0755 ${PG_RUNDIR} 57 | chmod g+s ${PG_RUNDIR} 58 | chown -R ${PG_USER}:${PG_USER} ${PG_RUNDIR} 59 | } 60 | 61 | set_postgresql_param() { 62 | local key=${1} 63 | local value=${2} 64 | local verbosity=${3:-verbose} 65 | 66 | if [[ -n ${value} ]]; then 67 | local current=$(exec_as_postgres sed -n -e "s/^\(${key} = '\)\([^ ']*\)\(.*\)$/\2/p" ${PG_CONF}) 68 | if [[ "${current}" != "${value}" ]]; then 69 | if [[ ${verbosity} == verbose ]]; then 70 | echo "Setting postgresql.conf parameter: ${key} = '${value}'" 71 | fi 72 | value="$(echo "${value}" | sed 's|[&]|\\&|g')" 73 | exec_as_postgres sed -i "s|^[#]*[ ]*${key} = .*|${key} = '${value}'|" ${PG_CONF} 74 | fi 75 | fi 76 | } 77 | 78 | set_recovery_param() { 79 | local key=${1} 80 | local value=${2} 81 | local hide=${3} 82 | if [[ -n ${value} ]]; then 83 | local current=$(exec_as_postgres sed -n -e "s/^\(.*\)\(${key}=\)\([^ ']*\)\(.*\)$/\3/p" ${PG_RECOVERY_CONF}) 84 | if [[ "${current}" != "${value}" ]]; then 85 | case ${hide} in 86 | true) echo "‣ Setting primary_conninfo parameter: ${key}" ;; 87 | *) echo "‣ Setting primary_conninfo parameter: ${key} = '${value}'" ;; 88 | esac 89 | exec_as_postgres sed -i "s|${key}=[^ ']*|${key}=${value}|" ${PG_RECOVERY_CONF} 90 | fi 91 | fi 92 | } 93 | 94 | set_hba_param() { 95 | local value=${1} 96 | if ! grep -q "$(sed "s| | \\\+|g" <<< ${value})" ${PG_HBA_CONF}; then 97 | echo "${value}" >> ${PG_HBA_CONF} 98 | fi 99 | } 100 | 101 | configure_ssl() { 102 | ## NOT SURE IF THIS IS A GOOD ALTERNATIVE TO ENABLE SSL SUPPORT BY DEFAULT ## 103 | ## BECAUSE USERS WHO PULL A PREBUILT IMAGE WILL HAVE THE SAME CERTIFICATES ## 104 | # if [[ ! -f ${PG_CERTDIR}/server.crt && ! -f ${PG_CERTDIR}/server.key ]]; then 105 | # if [[ -f /etc/ssl/certs/ssl-cert-snakeoil.pem && -f /etc/ssl/private/ssl-cert-snakeoil.key ]]; then 106 | # ln -sf /etc/ssl/certs/ssl-cert-snakeoil.pem ${PG_CERTDIR}/server.crt 107 | # ln -sf /etc/ssl/private/ssl-cert-snakeoil.key ${PG_CERTDIR}/server.key 108 | # fi 109 | # fi 110 | 111 | if [[ -f ${PG_CERTDIR}/server.crt && -f ${PG_CERTDIR}/server.key ]]; then 112 | PG_SSL=${PG_SSL:-on} 113 | set_postgresql_param "ssl_cert_file" "${PG_CERTDIR}/server.crt" 114 | set_postgresql_param "ssl_key_file" "${PG_CERTDIR}/server.key" 115 | fi 116 | PG_SSL=${PG_SSL:-off} 117 | set_postgresql_param "ssl" "${PG_SSL}" 118 | } 119 | 120 | configure_hot_standby() { 121 | case ${REPLICATION_MODE} in 122 | slave|snapshot|backup) ;; 123 | *) 124 | echo "Configuring hot standby..." 125 | set_postgresql_param "wal_level" "hot_standby" 126 | set_postgresql_param "max_wal_senders" "16" 127 | set_postgresql_param "checkpoint_segments" "8" 128 | set_postgresql_param "wal_keep_segments" "32" 129 | set_postgresql_param "hot_standby" "on" 130 | ;; 131 | esac 132 | } 133 | 134 | initialize_database() { 135 | if [[ ! -f ${PG_DATADIR}/PG_VERSION ]]; then 136 | case ${REPLICATION_MODE} in 137 | slave|snapshot|backup) 138 | if [[ -z $REPLICATION_HOST ]]; then 139 | echo "ERROR! Cannot continue without the REPLICATION_HOST. Exiting..." 140 | exit 1 141 | fi 142 | 143 | if [[ -z $REPLICATION_USER ]]; then 144 | echo "ERROR! Cannot continue without the REPLICATION_USER. Exiting..." 145 | exit 1 146 | fi 147 | 148 | if [[ -z $REPLICATION_PASS ]]; then 149 | echo "ERROR! Cannot continue without the REPLICATION_PASS. Exiting..." 150 | exit 1 151 | fi 152 | 153 | echo -n "Waiting for $REPLICATION_HOST to accept connections (60s timeout)" 154 | timeout=60 155 | while ! ${PG_BINDIR}/pg_isready -h $REPLICATION_HOST -p $REPLICATION_PORT -t 1 >/dev/null 2>&1 156 | do 157 | timeout=$(expr $timeout - 1) 158 | if [[ $timeout -eq 0 ]]; then 159 | echo "Timeout! Exiting..." 160 | exit 1 161 | fi 162 | echo -n "." 163 | sleep 1 164 | done 165 | echo 166 | 167 | case ${REPLICATION_MODE} in 168 | slave) 169 | echo "Replicating initial data from $REPLICATION_HOST..." 170 | exec_as_postgres PGPASSWORD=$REPLICATION_PASS ${PG_BINDIR}/pg_basebackup -D ${PG_DATADIR} \ 171 | -h ${REPLICATION_HOST} -p ${REPLICATION_PORT} -U ${REPLICATION_USER} -X stream -w >/dev/null 172 | ;; 173 | snapshot) 174 | echo "Generating a snapshot data on $REPLICATION_HOST..." 175 | exec_as_postgres PGPASSWORD=$REPLICATION_PASS ${PG_BINDIR}/pg_basebackup -D ${PG_DATADIR} \ 176 | -h ${REPLICATION_HOST} -p ${REPLICATION_PORT} -U ${REPLICATION_USER} -X fetch -w >/dev/null 177 | ;; 178 | backup) 179 | echo "Backing up data on $REPLICATION_HOST..." 180 | exec_as_postgres PGPASSWORD=$REPLICATION_PASS ${PG_BINDIR}/pg_basebackup -D ${PG_DATADIR} \ 181 | -h ${REPLICATION_HOST} -p ${REPLICATION_PORT} -U ${REPLICATION_USER} -X fetch -w >/dev/null 182 | exit 0 183 | ;; 184 | esac 185 | ;; 186 | *) 187 | echo "Initializing database..." 188 | PG_OLD_VERSION=$(find ${PG_HOME}/[0-9].[0-9]/main -maxdepth 1 -name PG_VERSION 2>/dev/null | grep -v $PG_VERSION | sort -r | head -n1 | cut -d'/' -f5) 189 | if [[ -n ${PG_OLD_VERSION} ]]; then 190 | echo "‣ Migrating PostgreSQL ${PG_OLD_VERSION} data to ${PG_VERSION}..." 191 | 192 | # protect the existing data from being altered by apt-get 193 | mv ${PG_HOME}/${PG_OLD_VERSION} ${PG_HOME}/${PG_OLD_VERSION}.migrating 194 | 195 | echo "‣ Installing PostgreSQL ${PG_OLD_VERSION}..." 196 | if ! ( apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql-pro-1c-${PG_OLD_VERSION} postgresql-client-pro-1c-${PG_OLD_VERSION} ) >/dev/null; then 197 | echo "ERROR! Failed to install PostgreSQL ${PG_OLD_VERSION}. Exiting..." 198 | # first move the old data back 199 | rm -rf ${PG_HOME}/${PG_OLD_VERSION} 200 | mv ${PG_HOME}/${PG_OLD_VERSION}.migrating ${PG_HOME}/${PG_OLD_VERSION} 201 | exit 1 202 | fi 203 | rm -rf /var/lib/apt/lists/* 204 | 205 | # we're ready to migrate, move back the old data and remove the trap 206 | rm -rf ${PG_HOME}/${PG_OLD_VERSION} 207 | mv ${PG_HOME}/${PG_OLD_VERSION}.migrating ${PG_HOME}/${PG_OLD_VERSION} 208 | fi 209 | 210 | if [[ -n $PG_PASSWORD ]]; then 211 | echo "${PG_PASSWORD}" > /tmp/pwfile 212 | fi 213 | 214 | exec_as_postgres ${PG_BINDIR}/initdb --pgdata=${PG_DATADIR} \ 215 | --username=${PG_USER} --encoding=unicode --auth=trust ${PG_PASSWORD:+--pwfile=/tmp/pwfile} >/dev/null 216 | 217 | if [[ -n ${PG_OLD_VERSION} ]]; then 218 | PG_OLD_BINDIR=/usr/lib/postgresql/${PG_OLD_VERSION}/bin 219 | PG_OLD_DATADIR=${PG_HOME}/${PG_OLD_VERSION}/main 220 | PG_OLD_CONF=${PG_OLD_DATADIR}/postgresql.conf 221 | PG_OLD_HBA_CONF=${PG_OLD_DATADIR}/pg_hba.conf 222 | PG_OLD_IDENT_CONF=${PG_OLD_DATADIR}/pg_ident.conf 223 | 224 | echo -n "‣ Migration in progress. Please be patient..." 225 | exec_as_postgres ${PG_BINDIR}/pg_upgrade \ 226 | -b ${PG_OLD_BINDIR} -B ${PG_BINDIR} \ 227 | -d ${PG_OLD_DATADIR} -D ${PG_DATADIR} \ 228 | -o "-c config_file=${PG_OLD_CONF} --hba_file=${PG_OLD_HBA_CONF} --ident_file=${PG_OLD_IDENT_CONF}" \ 229 | -O "-c config_file=${PG_CONF} --hba_file=${PG_HBA_CONF} --ident_file=${PG_IDENT_CONF}" >/dev/null 230 | echo 231 | fi 232 | ;; 233 | esac 234 | 235 | configure_hot_standby 236 | 237 | # Change DSM from `posix' to `sysv' if we are inside an lx-brand container 238 | if [[ $(uname -v) == "BrandZ virtual linux" ]]; then 239 | set_postgresql_param "dynamic_shared_memory_type" "sysv" 240 | fi 241 | fi 242 | 243 | # configure path to data_directory 244 | set_postgresql_param "data_directory" "${PG_DATADIR}" 245 | 246 | # configure logging 247 | set_postgresql_param "log_directory" "${PG_LOGDIR}" 248 | # set_postgresql_param "log_filename" "postgresql-${PG_VERSION}-main.log" 249 | 250 | # trust connections from local network 251 | if [[ ${PG_TRUST_LOCALNET} == true ]]; then 252 | echo "Trusting connections from the local network..." 253 | set_hba_param "host all all samenet trust" 254 | fi 255 | 256 | # allow remote connections to postgresql database 257 | set_hba_param "host all all 0.0.0.0/0 md5" 258 | } 259 | 260 | set_resolvconf_perms() { 261 | echo "Setting resolv.conf ACLs..." 262 | setfacl -m user:${PG_USER}:r /etc/resolv.conf || true 263 | } 264 | 265 | configure_recovery() { 266 | if [[ ${REPLICATION_MODE} == slave ]]; then 267 | echo "Configuring recovery..." 268 | if [[ ! -f ${PG_RECOVERY_CONF} ]]; then 269 | # initialize recovery.conf on the firstrun (slave only) 270 | exec_as_postgres touch ${PG_RECOVERY_CONF} 271 | ( echo "standby_mode = 'on'"; 272 | echo "primary_conninfo = 'host=${REPLICATION_HOST} port=${REPLICATION_PORT} user=${REPLICATION_USER} password=${REPLICATION_PASS} sslmode=${REPLICATION_SSLMODE}'"; 273 | ) > ${PG_RECOVERY_CONF} 274 | else 275 | set_recovery_param "host" "${REPLICATION_HOST}" 276 | set_recovery_param "port" "${REPLICATION_PORT}" 277 | set_recovery_param "user" "${REPLICATION_USER}" 278 | set_recovery_param "password" "${REPLICATION_PASS}" "true" 279 | set_recovery_param "sslmode" "${REPLICATION_SSLMODE}" 280 | fi 281 | else 282 | # recovery.conf can only exist on a slave node, its existence otherwise causes problems 283 | rm -rf ${PG_RECOVERY_CONF} 284 | fi 285 | } 286 | 287 | create_user() { 288 | if [[ -n ${DB_USER} ]]; then 289 | case $REPLICATION_MODE in 290 | slave|snapshot|backup) 291 | echo "INFO! Database user cannot be created on a $REPLICATION_MODE node. Skipping..." 292 | ;; 293 | *) 294 | if [[ -z ${DB_PASS} ]]; then 295 | echo "ERROR! Please specify a password for DB_USER in DB_PASS. Exiting..." 296 | exit 1 297 | fi 298 | echo "Creating database user: ${DB_USER}" 299 | if [[ -z $(psql -U ${PG_USER} -Atc "SELECT 1 FROM pg_catalog.pg_user WHERE usename = '${DB_USER}'";) ]]; then 300 | psql -U ${PG_USER} -c "CREATE ROLE \"${DB_USER}\" with LOGIN CREATEDB PASSWORD '${DB_PASS}';" >/dev/null 301 | fi 302 | ;; 303 | esac 304 | fi 305 | } 306 | 307 | setup_default_extension() { 308 | 309 | case $REPLICATION_MODE in 310 | slave|snapshot|backup) 311 | echo "INFO! Extensions cannot be created on a $REPLICATION_MODE node. Skipping..." 312 | ;; 313 | *) 314 | 315 | echo "SET Shared Libs" 316 | 317 | set_postgresql_param "shared_preload_libraries" "pg_stat_statements,powa,pg_qualstats,pg_stat_kcache" 318 | 319 | set_postgresql_param "powa.frequency" "1min" 320 | set_postgresql_param "powa.retention" "7d" 321 | set_postgresql_param "powa.database" "${POWA_DATABASE}" 322 | set_postgresql_param "powa.coalesce" "1000" 323 | 324 | echo "LOADING default extensions" 325 | 326 | psql -U ${PG_USER} -d ${DB_TEMPLATE} -c "UPDATE pg_language SET lanpltrusted = true WHERE lanname LIKE 'c';" >/dev/null 2>&1 327 | 328 | psql -U ${PG_USER} -d ${DB_TEMPLATE} -c "CREATE EXTENSION IF NOT EXISTS unaccent;" >/dev/null 2>&1 329 | psql -U ${PG_USER} -d ${DB_TEMPLATE} -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" >/dev/null 2>&1 330 | psql -U ${PG_USER} -d ${DB_TEMPLATE} -c "CREATE EXTENSION IF NOT EXISTS pgstattuple;" >/dev/null 2>&1 331 | psql -U ${PG_USER} -d ${DB_TEMPLATE} -c "CREATE EXTENSION IF NOT EXISTS pg_prewarm;" >/dev/null 2>&1 332 | psql -U ${PG_USER} -d ${DB_TEMPLATE} -c "CREATE EXTENSION IF NOT EXISTS pg_buffercache;" >/dev/null 2>&1 333 | psql -U ${PG_USER} -d ${DB_TEMPLATE} -c "CREATE EXTENSION IF NOT EXISTS pg_repack;" >/dev/null 2>&1 334 | psql -U ${PG_USER} -d ${DB_TEMPLATE} -c "CREATE EXTENSION IF NOT EXISTS hypopg;" >/dev/null 2>&1 335 | ;; 336 | esac 337 | 338 | } 339 | 340 | create_tablespaces() { 341 | 342 | case $REPLICATION_MODE in 343 | slave|snapshot|backup) 344 | echo "INFO! Tablespaces cannot be created on a $REPLICATION_MODE node. Skipping..." 345 | ;; 346 | *) 347 | 348 | echo "Create table spaces if not exist" 349 | 350 | psql -U ${PG_USER} -c "select public.make_tablespace('tmptblspc','${PG_TEMPTBLSPC}','');" 351 | psql -U ${PG_USER} -c "select public.make_tablespace('v81c_data','${PG_V81C_DATA}','');" 352 | psql -U ${PG_USER} -c "select public.make_tablespace('v81c_index','${PG_V81C_INDEX}','');" 353 | 354 | echo "SET temp_tablespaces" 355 | 356 | set_postgresql_param "temp_tablespaces" "tmptblspc" 357 | ;; 358 | esac 359 | 360 | } 361 | 362 | load_extensions() { 363 | local database=${1?missing argument} 364 | 365 | if [[ ${DB_UNACCENT} == true ]]; then 366 | echo 367 | echo "WARNING: " 368 | echo " The DB_UNACCENT option will be deprecated in favour of DB_EXTENSION soon." 369 | echo " Please migrate to using DB_EXTENSION" 370 | echo 371 | echo "‣ Loading unaccent extension..." 372 | psql -U ${PG_USER} -d ${database} -c "CREATE EXTENSION IF NOT EXISTS unaccent;" >/dev/null 2>&1 373 | fi 374 | 375 | for extension in $(awk -F',' '{for (i = 1 ; i <= NF ; i++) print $i}' <<< "${DB_EXTENSION}"); do 376 | echo "‣ Loading ${extension} extension..." 377 | psql -U ${PG_USER} -d ${database} -c "CREATE EXTENSION IF NOT EXISTS ${extension};" >/dev/null 2>&1 378 | done 379 | 380 | if [[ ${database} == ${POWA_DATABASE} ]]; then 381 | echo "‣ Loading POWA extensions..." 382 | psql -U ${PG_USER} -d ${database} -c "CREATE EXTENSION IF NOT EXISTS pg_stat_statements;" >/dev/null 2>&1 383 | psql -U ${PG_USER} -d ${database} -c "CREATE EXTENSION IF NOT EXISTS btree_gist;" >/dev/null 2>&1 384 | psql -U ${PG_USER} -d ${database} -c "CREATE EXTENSION IF NOT EXISTS pg_qualstats;" >/dev/null 2>&1 385 | psql -U ${PG_USER} -d ${database} -c "CREATE EXTENSION IF NOT EXISTS pg_stat_kcache;" >/dev/null 2>&1 386 | psql -U ${PG_USER} -d ${database} -c "CREATE EXTENSION IF NOT EXISTS pg_track_settings" >/dev/null 2>&1 387 | psql -U ${PG_USER} -d ${database} -c "CREATE EXTENSION IF NOT EXISTS powa" >/dev/null 2>&1 388 | 389 | fi 390 | } 391 | 392 | create_database() { 393 | if [[ -n ${DB_NAME} ]]; then 394 | case $REPLICATION_MODE in 395 | slave|snapshot|backup) 396 | echo "INFO! Database cannot be created on a $REPLICATION_MODE node. Skipping..." 397 | ;; 398 | *) 399 | for database in $(awk -F',' '{for (i = 1 ; i <= NF ; i++) print $i}' <<< "${DB_NAME}"); do 400 | echo "Creating database: ${database}..." 401 | if [[ -z $(psql -U ${PG_USER} -Atc "SELECT 1 FROM pg_catalog.pg_database WHERE datname = '${database}'";) ]]; then 402 | psql -U ${PG_USER} -c "CREATE DATABASE \"${database}\" WITH TEMPLATE = \"${DB_TEMPLATE}\";" >/dev/null 403 | fi 404 | 405 | load_extensions ${database} 406 | 407 | if [[ -n ${DB_USER} ]]; then 408 | echo "‣ Granting access to ${DB_USER} user..." 409 | psql -U ${PG_USER} -c "GRANT ALL PRIVILEGES ON DATABASE \"${database}\" to \"${DB_USER}\";" >/dev/null 410 | fi 411 | done 412 | ;; 413 | esac 414 | fi 415 | } 416 | 417 | create_replication_user() { 418 | if [[ -n ${REPLICATION_USER} ]]; then 419 | case $REPLICATION_MODE in 420 | slave|snapshot|backup) ;; # replication user can only be created on the master 421 | *) 422 | if [[ -z ${REPLICATION_PASS} ]]; then 423 | echo "ERROR! Please specify a password for REPLICATION_USER in REPLICATION_PASS. Exiting..." 424 | exit 1 425 | fi 426 | 427 | echo "Creating replication user: ${REPLICATION_USER}" 428 | if [[ -z $(psql -U ${PG_USER} -Atc "SELECT 1 FROM pg_catalog.pg_user WHERE usename = '${REPLICATION_USER}'";) ]]; then 429 | psql -U ${PG_USER} -c "CREATE ROLE \"${REPLICATION_USER}\" WITH REPLICATION LOGIN ENCRYPTED PASSWORD '${REPLICATION_PASS}';" >/dev/null 430 | fi 431 | 432 | set_hba_param "host replication ${REPLICATION_USER} 0.0.0.0/0 md5" 433 | ;; 434 | esac 435 | fi 436 | } 437 | 438 | set_max_connection() { 439 | # TODO - а зачем на реплике максимальное количество коннектов такое большое 440 | set_postgresql_param "max_connections" "${PG_MAX_CONNECTION}" 441 | echo "Set the maxconnection ${PG_MAX_CONNECTION}" 442 | } 443 | 444 | create_plsql_function() { 445 | case $REPLICATION_MODE in 446 | slave|snapshot|backup) 447 | echo "INFO! PLSQL cannot be created on a $REPLICATION_MODE node. Skipping..." 448 | ;; 449 | *) 450 | 451 | echo "Create functions PLSQL" 452 | 453 | psql -U ${PG_USER} -c "CREATE EXTENSION IF NOT EXISTS dblink;" >/dev/null 2>&1 454 | 455 | echo "Make tablespace if not exist" 456 | psql -U ${PG_USER} -f ${PG_APP_HOME}/plsql/make_tablespaces.pgsql 457 | 458 | ;; 459 | esac 460 | } 461 | 462 | set_timezone() { 463 | # а может ли реплика отличаться по временной зоне от мастера 464 | set_postgresql_param "timezone" "${PG_TIMEZONE}" 465 | echo "Set timezone to ${PG_TIMEZONE}" 466 | } 467 | 468 | adoptive_shared_buffers() { 469 | shared=$(expr $physmem / 4) 470 | set_postgresql_param "shared_buffers" "${shared}MB" 471 | 472 | echo "Set the shared ${shared}" 473 | } 474 | 475 | adoptive_temp_buffers() { 476 | set_postgresql_param "temp_buffers" "256MB" #Hm TODO 477 | echo "Set the temp 256Mb" 478 | } 479 | 480 | adoptive_autovacuum_workers() { 481 | set_postgresql_param "autovacuum_max_workers" "4" #Max(Core/2,4) TODO 482 | echo "Set the autovac-workers 4" 483 | } 484 | 485 | adoptive_work_mem() { 486 | local devmem=$(expr $physmem - $shared ) 487 | local conect=${PG_MAX_CONNECTION} 488 | local devider=$(expr 6) 489 | local mixed=$(expr $devider '*' $conect ) #mixed-type TODO 490 | work_mem=$(expr $devmem / $mixed ) 491 | set_postgresql_param "work_mem" "${work_mem}MB" #RAM/MaxConnection 492 | 493 | echo "Set the work_mem ${work_mem}" 494 | 495 | } 496 | 497 | adoptive_maintenance_mem() { 498 | local maintenance_work_mem=$(expr $work_mem '*' 4 ) 499 | set_postgresql_param "maintenance_work_mem" "${maintenance_work_mem}MB" #TODO we need to get max index from pg_catatlog 500 | 501 | echo "Set the maintenance_work_mem ${maintenance_work_mem}" 502 | } 503 | 504 | adoptive_cashe_size() { 505 | local devmem=$(expr $physmem - $shared ) 506 | set_postgresql_param "effective_cache_size" "${devmem}MB" 507 | 508 | echo "Set the effective_cache_size ${effective_cache_size}" 509 | 510 | } 511 | 512 | adoptive_io() { 513 | #cat /sys/block/sda/queue/rotational 514 | #lsblk -d -o name,rota TODO 515 | echo "we use iostat and ome tricks" 516 | set_postgresql_param "effective_io_concurrency" "2" 517 | } 518 | 519 | adoptive_sync() { 520 | echo "disable syncron commit" 521 | set_postgresql_param "synchronous_commit" "off" 522 | } 523 | 524 | adoptive_fstat() { 525 | echo 'we need to fix fstat here' 526 | } 527 | 528 | enable_magic() { 529 | if [[ ${PG_TUNE_BY_MAMONSU} == true ]]; then 530 | echo "MamonSu try to tune PG" 531 | exec_as_postgres ${PG_BINDIR}/pg_ctl -D ${PG_DATADIR} -w start >/dev/null 532 | mamonsu tune 533 | psql -U ${PG_USER} -c "ALTER SYSTEM SET log_filename='postgresql-%H.log';" >/dev/null 2>&1 534 | psql -U ${PG_USER} -c "ALTER SYSTEM SET log_min_duration_statement='0';" >/dev/null 2>&1 535 | psql -U ${PG_USER} -c "ALTER SYSTEM SET log_error_verbosity='default';" >/dev/null 2>&1 536 | psql -U ${PG_USER} -c "ALTER SYSTEM SET log_rotation_age='1h';" >/dev/null 2>&1 537 | psql -U ${PG_USER} -c "ALTER SYSTEM SET log_truncate_on_rotation='on';" >/dev/null 2>&1 538 | psql -U ${PG_USER} -c "ALTER SYSTEM SET log_timezone='Europe/Moscow';" >/dev/null 2>&1 539 | psql -U ${PG_USER} -c "ALTER SYSTEM SET log_file_mode = '0655';" >/dev/null 2>&1 540 | exec_as_postgres ${PG_BINDIR}/pg_ctl -D ${PG_DATADIR} -w stop >/dev/null 541 | echo "end mamonsu worker" 542 | echo "whereis an additional conf" 543 | cat ${PG_DATADIR}/postgresql.auto.conf 544 | fi 545 | } 546 | 547 | configure_postgresql() { 548 | initialize_database 549 | configure_recovery 550 | configure_ssl 551 | 552 | # start postgres server internally for the creation of users and databases 553 | rm -rf ${PG_DATADIR}/postmaster.pid 554 | set_postgresql_param "listen_addresses" "127.0.0.1" quiet 555 | exec_as_postgres ${PG_BINDIR}/pg_ctl -D ${PG_DATADIR} -w start >/dev/null 556 | 557 | setup_default_extension 558 | create_plsql_function 559 | 560 | 561 | create_tablespaces 562 | 563 | 564 | # stop server for reload conf 565 | exec_as_postgres ${PG_BINDIR}/pg_ctl -D ${PG_DATADIR} -w stop >/dev/null 566 | exec_as_postgres ${PG_BINDIR}/pg_ctl -D ${PG_DATADIR} -w start >/dev/null 567 | 568 | create_user 569 | create_database 570 | create_replication_user 571 | 572 | # stop the postgres server 573 | exec_as_postgres ${PG_BINDIR}/pg_ctl -D ${PG_DATADIR} -w stop >/dev/null 574 | 575 | #tune 576 | physmem=$(free -m|awk '/^Mem:/{print $2}') 577 | 578 | case $REPLICATION_MODE in 579 | slave|snapshot|backup) 580 | echo "INFO! Tune for replica is not processed $REPLICATION_MODE node. Skipping..." 581 | ;; 582 | *) 583 | 584 | set_max_connection 585 | set_timezone 586 | 587 | adoptive_shared_buffers 588 | adoptive_temp_buffers 589 | adoptive_autovacuum_workers 590 | adoptive_work_mem 591 | adoptive_maintenance_mem 592 | adoptive_cashe_size 593 | adoptive_io 594 | adoptive_sync 595 | 596 | enable_magic 597 | 598 | #adoptive_fstat 599 | esac 600 | 601 | # listen on all interfaces 602 | set_postgresql_param "listen_addresses" "*" quiet 603 | } 604 | -------------------------------------------------------------------------------- /runtime/plsql/make_tablespaces.pgsql: -------------------------------------------------------------------------------- 1 | -- https://dba.stackexchange.com/questions/150148/how-to-create-tablespace-if-it-does-not-exist 2 | CREATE OR REPLACE FUNCTION make_tablespace(tablespace text, 3 | directory text, 4 | owner text) 5 | RETURNS void AS 6 | $func$ 7 | BEGIN 8 | IF tablespace <> '' THEN -- catches '' *and* NULL 9 | -- do nothing 10 | ELSE 11 | RAISE EXCEPTION 'No tablespace.'; 12 | END IF; 13 | 14 | IF EXISTS (SELECT 1 FROM pg_tablespace WHERE spcname = tablespace) THEN 15 | RAISE NOTICE 'Tablespace % already exists.', tablespace; 16 | ELSE 17 | IF directory <> '' THEN 18 | ELSE 19 | RAISE EXCEPTION 'No directory.'; 20 | END IF; 21 | 22 | IF owner <> '' THEN 23 | ELSE 24 | RAISE NOTICE 'No owner. Will be set to postgres'; 25 | owner = 'postgres'; 26 | END IF; 27 | 28 | PERFORM dblink_connect('dbname=postgres'::text); -- name of foreign server 29 | PERFORM dblink_exec(format('CREATE TABLESPACE %I OWNER %I LOCATION %L', tablespace, owner, directory)); 30 | RAISE NOTICE 'Tablespace % created.', tablespace; 31 | PERFORM dblink_disconnect(); 32 | END IF; 33 | END 34 | $func$ LANGUAGE plpgsql; 35 | -------------------------------------------------------------------------------- /runtime/warmup: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Try to warmup some tables" 4 | 5 | set -e 6 | source ${PG_APP_HOME}/env-defaults 7 | -------------------------------------------------------------------------------- /tools/compsess-status.os: -------------------------------------------------------------------------------- 1 | #Использовать sql 2 | 3 | Перем Соединение; 4 | перем ЗапросНАСжатие; 5 | Перем ТекстЗапросаСжатие; 6 | 7 | Функция ПолучитьIDТАблицы(_имяТаблицы) 8 | 9 | ЗапросПоИД = Новый Запрос(); 10 | ЗапросПоИД.УстановитьСоединение(Соединение); 11 | 12 | ЗапросПоИД.Текст = "SELECT CAST (c.oid as VARCHAR) as oids, 13 | | n.nspname, 14 | | c.relname 15 | | FROM pg_catalog.pg_class c 16 | | LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace 17 | | WHERE c.relname = '" + _имяТаблицы + "' 18 | | AND pg_catalog.pg_table_is_visible(c.oid) 19 | | ORDER BY 2, 3;"; 20 | 21 | ТЗИД = ЗапросПоИД.Выполнить().Выгрузить(); 22 | 23 | Для каждого строкаТЗ Из ТЗИД Цикл 24 | Возврат строкаТЗ[0]; 25 | КонецЦикла; 26 | 27 | КонецФункции 28 | 29 | 30 | Функция ПолучитьКолонкиТаблицы(идТаблицы) 31 | 32 | ЗапросПоИД = Новый Запрос(); 33 | ЗапросПоИД.УстановитьСоединение(Соединение); 34 | 35 | ЗапросПоИД.Текст = " 36 | |SELECT a.attname, pg_catalog.format_type(a.atttypid, a.atttypmod), 37 | |( 38 | | SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid) for 128) 39 | | FROM pg_catalog.pg_attrdef d 40 | | WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef 41 | |), 42 | |a.attnotnull, a.attnum, 43 | |( 44 | | SELECT c.collname FROM pg_catalog.pg_collation c, pg_catalog.pg_type t 45 | | WHERE c.oid = a.attcollation AND t.oid = a.atttypid AND a.attcollation <> t.typcollation 46 | |) AS attcollation, 47 | |NULL AS indexdef, 48 | |NULL AS attfdwoptions, 49 | |CAST (a.attstorage AS VARCHAR) as storagetype, 50 | |CASE 51 | | WHEN a.attstattarget=-1 THEN NULL ELSE a.attstattarget END AS attstattarget, 52 | |pg_catalog.col_description(a.attrelid, a.attnum) 53 | |FROM pg_catalog.pg_attribute a 54 | |WHERE a.attrelid = '" + идТаблицы + "' AND a.attnum > 0 AND NOT a.attisdropped 55 | |ORDER BY a.attnum;"; 56 | 57 | ТЗИД = ЗапросПоИД.Выполнить().Выгрузить(); 58 | 59 | Возврат ТЗИД; 60 | 61 | КонецФункции 62 | 63 | Процедура СжатьСтолбец(столбец, таблица) 64 | 65 | ТекстЗапросаСжатие = "" + ТекстЗапросаСжатие + " 66 | |ALTER TABLE " + таблица + " ALTER " + столбец + " SET STORAGE EXTENDED;"; 67 | 68 | КонецПроцедуры 69 | 70 | 71 | Процедура ВывестиТаблицу(тзЗначений, таблица) 72 | 73 | Для каждого строкаТЗ Из тзЗначений Цикл 74 | наименование = строкаТЗ["attname"]; 75 | типСжатия = строкаТЗ["storagetype"]; 76 | типСтолбца = строкаТЗ["format_type"]; 77 | 78 | СтрокаСообщения = "" + наименование + " " + строкаТЗ["storagetype"] + " " + строкаТЗ["format_type"]; 79 | 80 | Если типСжатия = "p" И типСтолбца <> "timestamp without time zone" И типСтолбца <> "integer" И типСтолбца <> "boolean" И типСтолбца <> "bigint" Тогда 81 | СтрокаСообщения = СтрокаСообщения + " СЖИМАЕТСЯ"; 82 | СжатьСтолбец(наименование, таблица); 83 | КонецЕсли; 84 | 85 | КонецЦикла; 86 | 87 | КонецПроцедуры 88 | 89 | Сообщить("Проверка статуса сжатия PG-1С и собственно попытка сжатия"); 90 | 91 | 92 | Соединение = Новый Соединение(); 93 | Соединение.ТипСУБД = Соединение.ТипыСУБД.PostgreSQL; 94 | Соединение.Сервер = "localhost"; 95 | Соединение.ИмяПользователя = "postgres"; 96 | Соединение.ИмяБазы = ПолучитьПеременнуюСреды("DBNAME"); 97 | Соединение.Пароль = ПолучитьПеременнуюСреды("PG_PASSWORD"); 98 | Соединение.Порт = 5432; 99 | 100 | 101 | Соединение.Открыть(); 102 | 103 | ЗапросКБазе = Новый Запрос(); 104 | ЗапросКБазе.УстановитьСоединение(Соединение); 105 | 106 | ЗапросКБазе.Текст = " 107 | |SELECT * FROM information_schema.tables WHERE table_schema = 'public' and table_type <> 'VIEW' ORDER BY table_name 108 | |"; 109 | 110 | ТЗ = ЗапросКБазе.Выполнить().Выгрузить(); 111 | 112 | Для Каждого колонка из ТЗ.колонки Цикл 113 | //Сообщить("" + колонка.Имя); 114 | КонецЦикла; 115 | 116 | сч = 0; 117 | количество = ТЗ.Количество(); 118 | 119 | Для каждого строкаТаблицы Из ТЗ Цикл 120 | 121 | Сообщить("Обработка " + сч + " из " + количество); 122 | имяТаблицы = строкаТаблицы[2]; 123 | идТаблицы = ПолучитьIDТАблицы(имяТаблицы); 124 | 125 | Сообщить("Таблица " + имяТаблицы + " с ИД: " + идТаблицы); 126 | 127 | тзКолонок = ПолучитьКолонкиТаблицы(идТаблицы); 128 | ВывестиТаблицу(тзКолонок, имяТаблицы); 129 | 130 | сч = сч + 1 ; 131 | КонецЦикла; 132 | 133 | 134 | ЗапросНАСжатие = Новый Запрос(); 135 | ЗапросНАСжатие.УстановитьСоединение(Соединение); 136 | 137 | ЗапросНАСжатие.Текст = ТекстЗапросаСжатие; 138 | 139 | Старт = ТекущаяУниверсальнаяДатаВМиллисекундах(); 140 | Сообщить("СтартСжатия - " + ТекущаяДата()); 141 | 142 | ЗапросНАСжатие.Выполнить(); 143 | Конец = ТекущаяУниверсальнаяДатаВМиллисекундах(); 144 | Сообщить("ОкончаниеСжатия = " + ТекущаяДата()); 145 | Сообщить("Скорость в секундах - " + ((Конец-Старт)/1000)); 146 | 147 | 148 | ЗапросНАСжатие.Текст = ""; 149 | ЗапросНАСжатие.Выполнить(); 150 | -------------------------------------------------------------------------------- /tools/postgrepinning: -------------------------------------------------------------------------------- 1 | Package: postgres* 2 | Pin: release l=*1C patches 3 | Pin-Priority: 900 4 | -------------------------------------------------------------------------------- /upgrade.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "UPGRADE sandbox to latest version" 4 | 5 | echo "ATTENTION !!! this will remove containers and create new" 6 | echo "please answers yes, if you stoped aplications what use PG for testing" 7 | echo "use this script only in sandbox" 8 | 9 | docker-compose build 10 | docker-compose pull 11 | docker-compose down 12 | 13 | docker-compose up -d 14 | --------------------------------------------------------------------------------