├── .dockerignore ├── .github └── workflows │ └── dockerimage.yml ├── CROSS-COMPILING.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md └── start-coroqnetd.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | *.md 2 | .git/ 3 | .github/ 4 | Makefile 5 | Dockerfile 6 | LICENSE 7 | -------------------------------------------------------------------------------- /.github/workflows/dockerimage.yml: -------------------------------------------------------------------------------- 1 | name: Docker Image CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | 7 | build: 8 | 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Build the Docker image 14 | run: make 15 | -------------------------------------------------------------------------------- /CROSS-COMPILING.md: -------------------------------------------------------------------------------- 1 | # Cross compiling for ARM 2 | 3 | See the arm64v8 branch for more information. 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:bookworm-slim 2 | 3 | LABEL description="Corosync Qdevice Network daemon" 4 | LABEL documentation="man:corosync-qnetd" 5 | 6 | # Install the proxmox repository signing key. 7 | ADD "https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg" \ 8 | /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg 9 | 10 | # Create the coroqnetd user and group, set the sticky bit on /var/run so 11 | # corosync-qnetd can create its runtime directory, and install the proxmox 12 | # corosync3 repository. Don't create the runtime directory yet since the user 13 | # may want to run with a different uid/gid. 14 | # 15 | # Then install corosync-qnetd from the Proxmox repos. Also don't generate 16 | # the certs here and instead create them when the container is first run. 17 | RUN adduser --quiet --system --disabled-login --no-create-home \ 18 | --home /etc/corosync/qnetd --group --uid=903 coroqnetd \ 19 | && chmod 1777 /var/run \ 20 | && chmod a+r /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg \ 21 | && echo "deb http://download.proxmox.com/debian/pve bookworm pve-no-subscription" \ 22 | > /etc/apt/sources.list.d/corosync3.list \ 23 | && mkdir -p /etc/corosync/qnetd/nssdb \ 24 | && touch /etc/corosync/qnetd/nssdb/cert9.db \ 25 | && apt-get update \ 26 | && apt-get install --no-install-recommends -y corosync-qnetd \ 27 | && rm -rf /etc/corosync /var/lib/apt/lists/* 28 | 29 | # Run the status command periodically to make sure everything is working 30 | # (This returns success if qnetd is running, even if nobody is connected) 31 | HEALTHCHECK CMD corosync-qnetd-tool -s 32 | 33 | # The Corosync user and group to run as (change with docker run -u) 34 | USER 903:903 35 | 36 | # Corosync settings 37 | VOLUME /etc/corosync 38 | 39 | # The Corosync qnet device port (TCP) 40 | EXPOSE 5403 41 | 42 | # The umask to use when creating new files/directories (default: 022) 43 | ENV UMASK= 44 | 45 | # You can also add qnetd arguments via this environment variable 46 | ENV COROSYNC_QNETD_OPTIONS= 47 | 48 | # The arguments to pass to corosync-qnetd (-f is always passed). 49 | # E.g. -s required -m 1 50 | CMD [] 51 | ENTRYPOINT ["start-coroqnetd"] 52 | 53 | COPY start-coroqnetd.sh /usr/local/bin/start-coroqnetd 54 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020-2023 Tim Schlueter 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: build 2 | 3 | ARGS ?= -t corosync-qnetd:latest 4 | 5 | build: 6 | docker build $(ARGS) . 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dockerized Corosync QNet Daemon 2 | 3 | Sets up a Corosync v3 QNet Daemon for use with Proxmox. 4 | 5 | [Docker Hub](https://hub.docker.com/r/modelrockettier/corosync-qnetd) 6 | 7 | This allows you to deploy an external voter on a server that is not running Proxmox (e.g. a NAS). 8 | The external voter mainly serves to break ties (e.g. if the cluster has an even number of nodes). 9 | 10 | See 11 | for more information. 12 | 13 | **NOTE: This container does not have an SSH server installed, so setting it up is a bit more 14 | involved than a simple `pvecm qdevice setup`.** 15 | 16 | ## Getting Started 17 | 18 | ### Prerequisites 19 | 20 | This assumes that you have at least 2 Proxmox nodes and a separate docker server. 21 | 22 | You will also need to set a few environment variables (or manually replace them with the 23 | appropriate values in the commands below): 24 | 25 | * `CLUSTER_NAME`: The name of your Proxmox cluster (must match the `cluster_name` key in 26 | `/etc/corosync/corosync.conf` on your Proxmox nodes. 27 | 28 | E.g. 29 | ``` 30 | CLUSTER_NAME=cluster1 31 | ``` 32 | 33 | * `PROXMOX_NODE`: The credentials to ssh into your first Proxmox node (usually user@host or 34 | user@ip). 35 | 36 | E.g. 37 | ``` 38 | PROXMOX_NODE=root@proxmox1 39 | ``` 40 | 41 | * `QNETD_DATA`: Where to store the corosync-qnetd config data on the docker host. 42 | 43 | E.g. 44 | ``` 45 | QNETD_DATA=/etc/corosync-data 46 | ``` 47 | 48 | ### Instructions 49 | 50 | The general flow is to set up the qnet device with 1 Proxmox node, then to add the rest of the 51 | Proxmox nodes afterwards. 52 | 53 | You will need to run some commands on the docker host and some on the initial Proxmox node. 54 | Instructions below are prefixed with **\[docker\]** and **\[proxmox\]** respectively depending on 55 | where they need to be run. 56 | 57 | It makes no difference which Proxmox node you pick for the initial set up, but being able to 58 | directly transfer files (e.g. via scp) between it and the docker host will make it easier. 59 | 60 | 1. **\[docker\]** Pull the docker corosync-qnetd container (or build it from this repo) 61 | ``` 62 | docker pull modelrockettier/corosync-qnetd 63 | ``` 64 | 65 | 2. **\[docker\]** Create and start the docker corosync-qnetd container: 66 | ``` 67 | docker run -d --name=qnetd --cap-drop=ALL -p 5403:5403 \ 68 | -v ${QNETD_DATA}:/etc/corosync modelrockettier/corosync-qnetd 69 | ``` 70 | 71 | 3. **\[docker\]** Copy the QNetd CA certificate to the first Proxmox node: 72 | ``` 73 | scp ${QNETD_DATA}/qnetd/nssdb/qnetd-cacert.crt \ 74 | ${PROXMOX_NODE}:/etc/pve/corosync/qdevice/net/nssdb/ 75 | ``` 76 | * NOTE: Since the Proxmox cluster already synchronizes everything in the `/etc/pve` directory, 77 | it's easiest to just copy the CA certificate there on 1 node, and it'll automatically 78 | propogate to all other nodes. This way you won't need to copy it over to the other nodes 79 | individually. 80 | 81 | 4. **\[proxmox\]** Install the corosync-qdevice package on the first Proxmox node: 82 | ``` 83 | apt-get install corosync-qdevice 84 | ``` 85 | 86 | 5. **\[proxmox\]** Start and enable the corosync-qdevice service on the first Proxmox node: 87 | ``` 88 | systemctl start corosync-qdevice 89 | systemctl enable corosync-qdevice 90 | ``` 91 | 92 | * NOTE: if enable fails, try deleting `/etc/init.d/corosync-qdevice` and try again. 93 | See 94 | 95 | 6. **\[proxmox\]** Initialize the corosync-qdevice certificate database on the first Proxmox node: 96 | ``` 97 | corosync-qdevice-net-certutil -i \ 98 | -c /etc/pve/corosync/qdevice/net/nssdb/qnetd-cacert.crt 99 | ``` 100 | 101 | 7. **\[proxmox\]** Generate a certificate signing request on the first Proxmox node: 102 | ``` 103 | corosync-qdevice-net-certutil -r -n ${CLUSTER_NAME} 104 | ``` 105 | 106 | 8. **\[docker\]** Copy the certificate signing request to the corosync config directory: 107 | ``` 108 | scp ${PROXMOX_NODE}:/etc/corosync/qdevice/net/nssdb/qdevice-net-node.crq \ 109 | ${QNETD_DATA}/qnetd/nssdb/ 110 | ``` 111 | 112 | 9. **\[docker\]** Sign the certificate from the corosync-qnetd container: 113 | ``` 114 | docker exec qnetd \ 115 | corosync-qnetd-certutil -s -n ${CLUSTER_NAME} \ 116 | -c /etc/corosync/qnetd/nssdb/qdevice-net-node.crq 117 | ``` 118 | * Note the path to the .crq is from inside the container. 119 | 120 | 10. **\[docker\]** Copy the newly generated certificate back to the first Proxmox node: 121 | ``` 122 | scp ${QNETD_DATA}/qnetd/nssdb/cluster-${CLUSTER_NAME}.crt \ 123 | ${PROXMOX_NODE}:/etc/pve/corosync/qdevice/net/nssdb/ 124 | ``` 125 | 126 | 11. **\[proxmox\]** Import the certificate on the first Proxmox node: 127 | ``` 128 | corosync-qdevice-net-certutil -M -c cluster-${CLUSTER_NAME}.crt 129 | ``` 130 | 131 | 12. **\[proxmox\]** Copy the output qdevice-net-node.p12 to all other Proxmox nodes: 132 | ``` 133 | cp -v /etc/corosync/qdevice/net/nssdb/qdevice-net-node.p12 \ 134 | /etc/pve/corosync/qdevice/net/nssdb/ 135 | ``` 136 | * NOTE: We're again using the `/etc/pve` synchronization discussed in step 3. 137 | 138 | 13. **\[proxmox\]** Set up all other Proxmox nodes 139 | 1. Repeat steps 4-6 above 140 | 1. Install the corosync-qdevice package 141 | 142 | 2. Start and enable the corosync-qdevice service 143 | 144 | 3. Initialize the corosync-qdevice certificate database 145 | ``` 146 | corosync-qdevice-net-certutil -i \ 147 | -c /etc/pve/corosync/qdevice/net/nssdb/qnetd-cacert.crt 148 | ``` 149 | 150 | 2. Import the corosync cluster certificate and key: 151 | ``` 152 | corosync-qdevice-net-certutil -m \ 153 | -c /etc/pve/corosync/qdevice/net/nssdb/qdevice-net-node.p12 154 | ``` 155 | 156 | 14. **\[proxmox\]** Add qdevice config to `/etc/pve/corosync.conf` on the first Proxmox node: 157 | 158 | Edit `/etc/pve/corosync.conf`: 159 | * Replace: 160 | ``` 161 | quorum { 162 | provider: corosync_votequorum 163 | } 164 | ``` 165 | * With the following (and change `${DOCKER_HOST}` to the hostname or IP of your docker host): 166 | ``` 167 | quorum { 168 | provider: corosync_votequorum 169 | device { 170 | model: net 171 | votes: 1 172 | net { 173 | tls: on 174 | host: ${DOCKER_HOST} 175 | algorithm: ffsplit 176 | } 177 | } 178 | } 179 | ``` 180 | See for more info. 181 | 182 | 15. **\[proxmox\]** Restart the corosync-qdevice service **on all Proxmox nodes**: 183 | ``` 184 | systemctl restart corosync-qdevice 185 | ``` 186 | 187 | 16. **\[docker\]** [Ensure corosync-qnetd is working properly] 188 | 189 | The number of connected clients should be equal to the number of proxmox nodes online. 190 | 191 | ## Quick Setup (untested/unsupported) 192 | 193 | This should work and be a bit quicker and easier than the above quick start guide, but it hasn't 194 | been tested and requires your proxmox nodes to be able to SSH into your docker host. 195 | 196 | ### Prerequisites 197 | 198 | Your docker host **must** have an SSH server installed and the Proxmox node used in step 5 199 | **must** be able to SSH into your docker server. 200 | 201 | You will also need to set a few environment variables (or manually replace them with the 202 | appropriate values in the commands below): 203 | 204 | * `CLUSTER_NAME`: The name of your Proxmox cluster (must match the `cluster_name` key in 205 | `/etc/corosync/corosync.conf` on your Proxmox nodes. 206 | 207 | E.g. 208 | ``` 209 | CLUSTER_NAME=pm-cluster-1 210 | ``` 211 | 212 | * `DOCKER_HOST`: The hostname or IP address of your docker host 213 | 214 | E.g. 215 | ``` 216 | DOCKER_HOST=docker1 217 | ``` 218 | 219 | * `PROXMOX_NODES`: The hostnames or IP addresses of your proxmox nodes 220 | 221 | E.g. 222 | ``` 223 | PROXMOX_NODES=( proxmox1 proxmox2 ) 224 | ``` 225 | 226 | ### Instructions 227 | 228 | 1. On all Proxmox nodes, install the corosync-qdevice package: 229 | ``` 230 | apt-get install corosync-qdevice 231 | ``` 232 | 233 | 2. On all Proxmox nodes, start and enable the corosync-qdevice service: 234 | ``` 235 | systemctl start corosync-qdevice 236 | systemctl enable corosync-qdevice 237 | ``` 238 | 239 | 3. On the docker host, create and start the docker corosync-qnetd container: 240 | ``` 241 | docker run -d --name=qnetd --cap-drop=ALL -p 5403:5403 \ 242 | -v /etc/corosync:/etc/corosync modelrockettier/corosync-qnetd 243 | ``` 244 | * NOTE: The corosync-qnetd data **must** be stored in `/etc/corosync` on the docker host. 245 | 246 | 4. On the docker host, copy the QNetd tools into the `$PATH`: 247 | ``` 248 | sudo docker cp qnetd:/usr/bin/corosync-qnetd-tool /usr/local/bin/ 249 | sudo docker cp qnetd:/usr/bin/corosync-qnetd-certutil /usr/local/bin/ 250 | ``` 251 | 252 | 5. From a Proxmox node, run the Proxmox cluster qdevice setup 253 | ``` 254 | pvecm qdevice setup ${DOCKER_HOST} 255 | ``` 256 | 257 | - The `corosync-qdevice-net-certutil` quick setup may also work (again, this is untested). 258 | ``` 259 | corosync-qdevice-net-certutil -Q -n ${CLUSTER_NAME} 260 | ${DOCKER_HOST} ${PROXMOX_NODES[@]} 261 | ``` 262 | 263 | 6. On the docker host, [Ensure corosync-qnetd is working properly] 264 | 265 | The number of connected clients should be equal to the number of proxmox nodes online. 266 | 267 | [Ensure corosync-qnetd is working properly]: <#check-on-a-running-corosync-qnet-daemon> 268 | 269 | ## Check on a running Corosync QNet Daemon 270 | 271 | ``` 272 | docker exec qnetd corosync-qnetd-tool -s 273 | ``` 274 | 275 | #### Example output 276 | 277 | ``` 278 | QNetd address: *:5403 279 | TLS: Supported (client certificate required) 280 | Connected clients: 2 281 | Connected clusters: 1 282 | ``` 283 | -------------------------------------------------------------------------------- /start-coroqnetd.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | dir=/etc/corosync/qnetd 4 | db="$dir/nssdb" 5 | 6 | set -e 7 | 8 | umask ${UMASK:-022} 9 | 10 | if [ ! -d "$db" ]; then 11 | echo "Creating corosync database dir: $db" 12 | mkdir -p "$db" 13 | fi 14 | 15 | ## Generate the corosync certificates 16 | ## (taken from the corosync-qnetd v3 debian postinst script) 17 | if [ ! -f "$db/cert9.db" ]; then 18 | if [ -f "$db/cert8.db" ]; then 19 | echo "Upgrading corosync certificate database" 20 | # if the password file is empty, add a newline so it's accepted 21 | if [ -f "$db/pwdfile.txt" -a ! -s "$db/pwdfile.txt" ]; then 22 | echo > "$db/pwdfile.txt" 23 | fi 24 | 25 | # upgrade to SQLite database 26 | certutil -N -d "sql:$db" -f "$db/pwdfile.txt" -@ "$db/pwdfile.txt" 27 | # Make cert9.db and key4.db permissions the same as cert8.db's perms 28 | chmod --reference="$db/cert8.db" "$db/cert9.db" "$db/key4.db" 29 | else 30 | echo "Creating corosync certificates" 31 | corosync-qnetd-certutil -i -G 32 | fi 33 | fi 34 | 35 | if [ -n "$COROSYNC_QNETD_OPTIONS$*" ]; then 36 | echo "Starting corosync-qnetd with args: $COROSYNC_QNETD_OPTIONS $*" 37 | else 38 | echo "Starting corosync-qnetd" 39 | fi 40 | error=0 41 | exec /usr/bin/corosync-qnetd -f $COROSYNC_QNETD_OPTIONS "$@" || error=$? 42 | 43 | echo "Failed to start corosync-qnetd: $error" >&2 44 | # exec somehow failed, return the error code 45 | exit $error 46 | 47 | ## Original systemd service: 48 | #[Service] 49 | #EnvironmentFile=-/etc/default/corosync-qnetd 50 | #ExecStart=/usr/bin/corosync-qnetd -f $COROSYNC_QNETD_OPTIONS 51 | #Type=notify 52 | #StandardError=null 53 | #Restart=on-abnormal 54 | #User=coroqnetd 55 | #RuntimeDirectory=corosync-qnetd 56 | #RuntimeDirectoryMode=0770 57 | #PrivateTmp=yes 58 | --------------------------------------------------------------------------------