├── .gitignore ├── .gitmodules ├── Makefile ├── README.md ├── buildout.sh ├── cloud_monitor.py3 ├── grafana ├── Dockerfile ├── dashboards.yaml ├── grafana.ini └── influxdb.yaml ├── influxdb ├── Dockerfile ├── influxdb.conf └── setup.sh ├── keycloak ├── Dockerfile └── startup.sh ├── open-ondemand ├── Dockerfile ├── entrypoint.sh ├── ood_portal.yml ├── sbatch ├── scaleout.yaml ├── scancel ├── scontrol └── squeue ├── proxy ├── Dockerfile ├── auth.php ├── nginx.conf ├── validate.php └── www.conf ├── scaleout ├── Dockerfile ├── bashrc ├── benchmark │ ├── compile.d │ │ └── utmost.sh │ └── run.d │ │ └── utmost.sh ├── dump_xdmod.sh ├── enroot.conf ├── get_keycloak_jwt.sh ├── globals.local ├── lab_scripts │ ├── PrologSlurmctld.sh │ ├── alloc.c │ ├── arrayjob.py │ ├── associations.json │ ├── canceljob.py │ ├── cgroup.conf │ ├── cleandemos.sh │ ├── cluster.cfg │ ├── dataprocessingjob.sh │ ├── dataprocessingprogram.sh │ ├── demo1.sh │ ├── demo2.sh │ ├── demo3.sh │ ├── depjob.py │ ├── eatmem.c │ ├── gen_jwt.py │ ├── gethostname.sh │ ├── hetjob.py │ ├── job.json │ ├── job.py │ ├── job_submit.lua │ ├── job_submit_spank.lua │ ├── make_notes.sh │ ├── memalloc.c │ ├── memalloc_with_sleep.c │ ├── myprogram.c │ ├── myslurmarray.sh │ ├── node_exporter.service │ ├── pi.c │ ├── plugstack.conf │ ├── prolog.sh │ ├── prometheus-slurm-exporter.service │ ├── prometheus.service │ ├── prometheus.update-1.yml │ ├── prometheus.update-2.yml │ ├── prometheus.yml │ ├── qos.sh │ ├── renice.c │ ├── restart.sh │ ├── sacct.py │ ├── sdiag.py │ ├── showjob.py │ ├── test.sh │ ├── testaccount.json │ ├── testping.sh.fred │ ├── testping.sh.pebbles │ ├── testuser.json │ ├── topology.conf │ ├── verify_jwt.py │ └── whereami.c ├── login.startup.sh ├── msmtprc ├── munge.service ├── my.cnf ├── mysql ├── patch.d │ └── .gitkeep ├── podman-containers │ ├── containers.conf │ ├── policy.json │ ├── registries.conf │ └── storage.conf ├── postfix.service ├── profile.sh ├── resume.node.sh ├── sackd.check.sh ├── sackd.service ├── slurm.bash_profile ├── slurm │ ├── acct_gather.conf │ ├── cgroup.conf │ ├── gres.conf │ ├── job_container.conf │ ├── nodes.conf │ ├── oci.conf │ ├── plugstack.conf │ ├── plugstack.conf.d │ │ └── README │ ├── scrun.lua │ ├── slurm.conf │ ├── slurm.jwt.conf │ ├── slurmdbd.conf │ └── staging.lua ├── slurmctld.service ├── slurmctld.startup.sh ├── slurmctld.startup2.sh ├── slurmd.check.sh ├── slurmd.service ├── slurmd.slice ├── slurmd.startup.sh ├── slurmdbd.service ├── slurmdbd.startup.sh ├── slurmrestd.env ├── slurmrestd.service ├── slurmrestd.startup.sh ├── startup.sh ├── suspend.node.sh ├── test-build.sh ├── testsuite.conf ├── tls_gen_csr.sh ├── tls_get_node_cert_key.sh ├── tls_get_node_token.sh ├── tls_sign_csr.sh ├── tls_validate_node.sh └── valgrind.patch ├── sql_server ├── Dockerfile └── my.cnf └── xdmod ├── Dockerfile ├── resource_specs.json ├── resources.json └── startup.sh /.gitignore: -------------------------------------------------------------------------------- 1 | docker-compose.yml 2 | scaleout/hosts.nodes 3 | scaleout/nodelist 4 | scaleout/patch.d/*.patch 5 | cloud_socket 6 | scaleout.tar 7 | federation 8 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/naterini/docker-scale-out/40c8a775c6e5cbb9461eb1cd218dbf9b3f1fe3e3/.gitmodules -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | HOST ?= login 2 | BUILD ?= up --build --remove-orphans -d 3 | DC ?= $(shell docker compose version 2>&1 >/dev/null && echo "docker compose" || echo "docker-compose") 4 | IMAGES := $(shell $(DC) config | awk '{if ($$1 == "image:") print $$2;}' | sort | uniq) 5 | SUBNET ?= 10.11 6 | SUBNET6 ?= 2001:db8:1:1:: 7 | 8 | .EXPORT_ALL_VARIABLES: 9 | 10 | default: ./docker-compose.yml run 11 | 12 | ./docker-compose.yml: buildout.sh 13 | bash buildout.sh > ./docker-compose.yml 14 | 15 | build: ./docker-compose.yml 16 | env COMPOSE_HTTP_TIMEOUT=3000 $(DC) --ansi=never --progress=plain $(BUILD) 17 | 18 | stop: 19 | $(DC) down 20 | 21 | set_nocache: 22 | $(eval BUILD := build --no-cache) 23 | 24 | nocache: set_nocache build 25 | 26 | clean-nodelist: 27 | truncate -s0 scaleout/nodelist 28 | 29 | clean: 30 | test -f ./docker-compose.yml && ($(DC) kill -s SIGKILL; $(DC) down --remove-orphans -t1 -v; unlink ./docker-compose.yml) || true 31 | [ -f cloud_socket ] && unlink cloud_socket || true 32 | 33 | uninstall: 34 | $(DC) down --rmi all --remove-orphans -t1 -v 35 | $(DC) rm -v 36 | 37 | run: ./docker-compose.yml 38 | $(DC) up --remove-orphans -d 39 | 40 | cloud: 41 | test -f cloud_socket && unlink cloud_socket || true 42 | touch cloud_socket 43 | test -f ./docker-compose.yml && unlink ./docker-compose.yml || true 44 | env CLOUD=1 bash buildout.sh > ./docker-compose.yml 45 | python3 ./cloud_monitor.py3 "$(DC)" 46 | test -f ./docker-compose.yml && unlink ./docker-compose.yml || true 47 | test -f cloud_socket && unlink cloud_socket || true 48 | 49 | bash: 50 | $(DC) exec $(HOST) /bin/bash 51 | 52 | save: build 53 | docker save -o scaleout.tar $(IMAGES) 54 | 55 | load: 56 | docker load -i scaleout.tar 57 | 58 | benchmark-%: clean-nodelist clean 59 | $(eval SLURM_BENCHMARK := $(subst benchmark-,,$@)) 60 | env SLURM_BENCHMARK=$(SLURM_BENCHMARK) bash buildout.sh > ./docker-compose.yml 61 | env COMPOSE_HTTP_TIMEOUT=3000 $(DC) --ansi=never --progress=plain $(BUILD) 62 | $(DC) up --remove-orphans -d 63 | $(DC) exec $(HOST) bash -c '(find /root/benchmark/run.d/ -type f -name $(SLURM_BENCHMARK)\*.sh | xargs -i echo bash "{} &"; echo wait) | bash -x' 64 | $(DC) down &>/dev/null 65 | truncate -s0 scaleout/nodelist 66 | 67 | test-build: clean-nodelist clean build 68 | $(DC) exec $(HOST) bash /usr/local/bin/test-build.sh 69 | test -f ./docker-compose.yml && ($(DC) kill -s SIGKILL; $(DC) down --remove-orphans -t1 -v; unlink ./docker-compose.yml) || true 70 | [ -f cloud_socket ] && unlink cloud_socket || true 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # slurm-docker-scaleout 2 | Docker compose cluster for testing Slurm 3 | 4 | ## Prerequisites 5 | * docker (25.x.x+ with cgroupsv2 or 24.x.x with cgroupsv1) 6 | * IPv6 must be configured in docker: https://docs.docker.com/config/daemon/ipv6/ 7 | * docker-compose-plugin v2.18.1+ 8 | * ssh (client) 9 | * jq 10 | * python3 11 | * python3-daemon 12 | 13 | ## Changes needed in sysctl.conf: 14 | ``` 15 | net.ipv4.tcp_max_syn_backlog=4096 16 | net.core.netdev_max_backlog=1000 17 | net.core.somaxconn=15000 18 | 19 | # Force gc to clean-up quickly 20 | net.ipv4.neigh.default.gc_interval = 3600 21 | 22 | # Set ARP cache entry timeout 23 | net.ipv4.neigh.default.gc_stale_time = 3600 24 | 25 | # Setup DNS threshold for arp 26 | net.ipv4.neigh.default.gc_thresh3 = 8096 27 | net.ipv4.neigh.default.gc_thresh2 = 4048 28 | net.ipv4.neigh.default.gc_thresh1 = 1024 29 | 30 | # Increase map count for elasticsearch 31 | vm.max_map_count=262144 32 | 33 | # Avoid running out of file descriptors 34 | fs.file-max=10000000 35 | fs.inotify.max_user_instances=65535 36 | fs.inotify.max_user_watches=1048576 37 | 38 | #Request kernel max number of cgroups 39 | fs.inotify.max_user_instances=65535 40 | ``` 41 | 42 | ## Docker configuration required with cgroupsv2 43 | 44 | Make sure the host machine is running CgroupV2 and not hybrid mode: 45 | https://slurm.schedmd.com/faq.html#cgroupv2 46 | 47 | Add these settings to the docker configuration: /etc/docker/daemon.json 48 | ``` 49 | { 50 | "exec-opts": [ 51 | "native.cgroupdriver=systemd" 52 | ], 53 | "features": { 54 | "buildkit": true 55 | }, 56 | "experimental": true, 57 | "cgroup-parent": "docker.slice", 58 | "default-cgroupns-mode": "host", 59 | "storage-driver": "overlay2" 60 | } 61 | ``` 62 | 63 | Configure systemd to allow docker to run in it's own slice to avoid systemd 64 | conflicting with it: 65 | 66 | /etc/systemd/system/docker.slice: 67 | ``` 68 | [Unit] 69 | Description=docker slice 70 | Before=slices.target 71 | [Slice] 72 | CPUAccounting=true 73 | CPUWeight=idle 74 | CPUQuota=90% 75 | MemoryAccounting=true 76 | MemoryMax=90% 77 | IOAccounting=true 78 | IOWeight=1 79 | ``` 80 | 81 | /etc/systemd/system/docker.service.d/local.conf: 82 | ``` 83 | [Unit] 84 | After=docker.slice 85 | Requires=docker.slice 86 | [Service] 87 | Slice=docker.slice 88 | ``` 89 | 90 | /usr/lib/systemd/system/docker.service.d/local.conf: 91 | ``` 92 | [Service] 93 | LimitNOFILE=infinity 94 | LimitNPROC=infinity 95 | LimitCORE=infinity 96 | TasksMax=infinity 97 | Delegate=yes 98 | ``` 99 | 100 | Activate the changes: 101 | ``` 102 | make clean 103 | sudo systemctl daemon-reload 104 | sudo systemctl restart docker.slice docker.service 105 | ``` 106 | 107 | Verify docker.slice is being used by docker: 108 | ``` 109 | make 110 | sudo systemctl status docker.slice docker.service 111 | ``` 112 | The container processes should now show up docker.slice process tree. 113 | 114 | ## Basic Architecture 115 | 116 | Maria Database Node: 117 | * db 118 | 119 | Slurm Management Nodes: 120 | * mgmtnode 121 | * mgmtnode2 122 | * slurmdbd 123 | 124 | Compute Nodes: 125 | * node[00-09] 126 | 127 | Login Nodes: 128 | * login 129 | 130 | Nginx Proxy node: 131 | * proxy 132 | 133 | Rest API Nodes: 134 | * rest 135 | 136 | Kibana (Only supports IPv4): 137 | * View http://127.0.0.1:5601/ 138 | 139 | Elasticsearch: 140 | * View http://localhost:9200/ 141 | 142 | Grafana: 143 | * View http://localhost:3000/ 144 | * User: admin 145 | * Password: admin 146 | 147 | Open On-Demand: 148 | * View http://localhost:8081/ 149 | * User: {user name - "fred" or "wilma"} 150 | * Password: password 151 | 152 | Open XDMoD: 153 | * View http://localhost:8082/ 154 | 155 | Proxy: 156 | * Auth REST API http://localhost:8080/auth 157 | * Query REST API http://localhost:8080/slurm/ 158 | 159 | Keycloak 160 | * Admin Console: http://127.0.0.1:8083/ 161 | * User: admin 162 | * Password: password 163 | 164 | ## Multiple Instances 165 | Each cluster must have a unique class B subnet. 166 | 167 | Default IPv4 is SUBNET="10.11". 168 | Default IPv6 is SUBNET6="2001:db8:1:1::". 169 | 170 | ## Custom Nodes 171 | 172 | Custom node lists may be provided by setting NODELIST to point to a file 173 | containing list of nodes for the cluster or modifying the default generated 174 | "nodelist" file in the scaleout directory. 175 | 176 | The node list follows the following format with one node per line: 177 | > ${HOSTNAME} ${CLUSTERNAME} ${IPv4} ${IPv6} 178 | 179 | Example line: 180 | > node00 scaleout 10.11.5.0 2001:db8:1:1::5:0 181 | 182 | Note that the service nodes can not be changed and will always be placed into 183 | the following subnets: 184 | > ${SUBNET}.1.0/24 185 | > ${SUBNET6}1:0/122 186 | 187 | ## Custom Slurm version 188 | 189 | To specify an explicit version of Slurm to be compiled and installed: 190 | > export SLURM_RELEASE=slurm-$version 191 | 192 | Make sure to call `make clean` after to invalidate all the caches with the 193 | prior release. 194 | 195 | ## To build images 196 | 197 | ``` 198 | git submodule update --init --force --remote --recursive 199 | make build 200 | ``` 201 | 202 | ## To run: 203 | 204 | ``` 205 | make 206 | ``` 207 | 208 | ## To build and run in Cloud mode: 209 | 210 | ``` 211 | make clean 212 | make cloud 213 | ``` 214 | 215 | Note: cloud mode will run in the foreground. 216 | 217 | ## To build without caching: 218 | 219 | ``` 220 | make nocache 221 | ``` 222 | 223 | ## To stop: 224 | 225 | ``` 226 | make stop 227 | ``` 228 | 229 | ## To reverse all changes: 230 | 231 | ``` 232 | make clean 233 | ``` 234 | 235 | ## To remove all images: 236 | 237 | ``` 238 | make uninstall 239 | ``` 240 | 241 | ## To control: 242 | 243 | ``` 244 | make bash 245 | make HOST=node1 bash 246 | ``` 247 | 248 | ## To login via ssh 249 | ``` 250 | ssh-keygen -f "/home/$(whoami)/.ssh/known_hosts" -R "10.11.1.5" 2>/dev/null 251 | ssh -o StrictHostKeyChecking=no -l fred 10.11.1.5 -X #use 'password' 252 | ``` 253 | 254 | ## Federation Mode 255 | 256 | Federation mode will create multiple Slurm clusters with nodes and slurmctld 257 | daemons. Other nodes will be shared, such as login and slurmdbd. 258 | 259 | To create multiple federation clusters: 260 | ``` 261 | export FEDERATION="taco burrito quesadilla" 262 | echo "FederationParameters=fed_display" >> scaleout/slurm/slurm.conf 263 | truncate -s0 scaleout/nodelist 264 | make clean 265 | make build 266 | make 267 | ``` 268 | 269 | Configure Slurm for multiple federation clusters: 270 | ``` 271 | make HOST=quesadilla-mgmtnode bash 272 | sacctmgr add federation scaleout clusters=taco,burrito,quesadilla 273 | ``` 274 | 275 | ### Activate Federation mode in Slurm 276 | 277 | Notify slurmdbd to use federation after building cluster: 278 | ``` 279 | export FEDERATION="taco burrito quesadilla" 280 | make HOST=taco-mgmtnode bash 281 | sacctmgr add federation scaleout cluster=taco,burrito,quesadilla 282 | ``` 283 | 284 | ### Deactivate to Federation mode 285 | 286 | ``` 287 | export FEDERATION="taco burrito quesadilla" 288 | make uninstall 289 | truncate -s0 scaleout/nodelist 290 | ``` 291 | 292 | ## Caveats 293 | 294 | The number of CPU threads on the host are multiplied by the number of nodes. Do not attempt to use computationally intensive applications. 295 | 296 | ## Docker work-arounds: 297 | 298 | ``` 299 | ERROR: Pool overlaps with other one on this address space 300 | ``` 301 | or 302 | ``` 303 | failed to prepare ${HASH}: max depth exceeded 304 | ERROR: Service 'slurmdbd' failed to build : Build failed 305 | ``` 306 | Call this: 307 | ``` 308 | make clean 309 | docker network prune -f 310 | sudo systemctl restart docker 311 | ``` 312 | 313 | ## To save all images to ./scaleout.tar 314 | 315 | ``` 316 | make save 317 | ``` 318 | 319 | ## To load saved copy of all images 320 | 321 | ``` 322 | make load 323 | ``` 324 | 325 | ## To test building 326 | 327 | ``` 328 | git submodule update --init --force --remote --recursive 329 | make test-build 330 | ``` 331 | 332 | ## How to trigger manual xdmod data dump: 333 | 334 | ``` 335 | make HOST=scaleout_mgmtnode_1 bash 336 | bash /etc/cron.hourly/dump_xdmod.sh 337 | exit 338 | make bash 339 | exec bash /etc/cron.hourly/dump_xdmod.sh 340 | make HOST=xdmod bash 341 | sudo -u xdmod -- /usr/bin/xdmod-shredder -r scaleout -f slurm -i /xdmod/data.csv 342 | sudo -u xdmod -- /usr/bin/xdmod-ingestor 343 | exit 344 | ``` 345 | 346 | ## How to disable building xdmod container 347 | 348 | This is will only disable attempts to build and start the container. 349 | 350 | ``` 351 | export DISABLE_XDMOD=1 352 | ``` 353 | 354 | ## How to disable building gdb 355 | 356 | This is will only disable attempts to build gdb from source. 357 | 358 | ``` 359 | export DISABLE_GDB_BUILD=1 360 | ``` 361 | 362 | ## How to disable building enroot+pyxis 363 | 364 | This is will only disable attempts to build enroot and pyxis from source. 365 | 366 | ``` 367 | export DISABLE_PYXIS=1 368 | ``` 369 | 370 | ## Maxing out kernel cgroups total 371 | 372 | The Linux kernel has a hard limit of 65535 cgroups total. Stacking large number 373 | of jobs or scaleout instances may result in the following error: 374 | 375 | ``` 376 | error: proctrack_g_create: No space left on device 377 | ``` 378 | 379 | When this happens, fewer jobs must be run as this a kernel limitation. 380 | -------------------------------------------------------------------------------- /buildout.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | unset MAC 3 | [[ $OSTYPE == 'darwin'* ]] && MAC=1 4 | 5 | #only mount cgroups with v1 6 | #https://github.com/jepsen-io/jepsen/issues/532#issuecomment-1128067136 7 | [ ! -f /sys/fs/cgroup/cgroup.controllers ] && SYSDFSMOUNTS=" 8 | - /etc/localtime:/etc/localtime:ro 9 | - /run/ 10 | - /run/lock/ 11 | - /sys/:/sys/:ro 12 | - /sys/fs/cgroup/:/sys/fs/cgroup/:ro 13 | - /sys/fs/fuse/:/sys/fs/fuse/:rw 14 | - /tmp/ 15 | - /var/lib/journal 16 | " || SYSDFSMOUNTS=" 17 | - /etc/localtime:/etc/localtime:ro 18 | - /run/ 19 | - /run/lock/ 20 | - /sys/ 21 | - /sys/fs/cgroup/:/sys/fs/cgroup/:ro 22 | - /sys/fs/cgroup/docker.slice/:/sys/fs/cgroup/docker.slice/:rw 23 | - /sys/fs/fuse/:/sys/fs/fuse/:rw 24 | - /tmp/ 25 | - /var/lib/journal 26 | " 27 | 28 | CACHE_DESTROYER="$(find scaleout/patch.d -type f -name '*.patch' -print0 | sort -z | xargs -0 cat | sha256sum | cut -b1-20)" 29 | 30 | SLURM_RELEASE="${SLURM_RELEASE:-master}" 31 | DISTRO="almalinux:8" 32 | if [ -z "$SUBNET" -o "$SUBNET" = "10.11" ] 33 | then 34 | ES_PORTS=" 35 | ports: 36 | - 9200:9200 37 | " 38 | KIBANA_PORTS=" 39 | ports: 40 | - 5601:5601 41 | " 42 | PROXY_PORTS=" 43 | ports: 44 | - 8080:8080 45 | " 46 | GRAFANA_PORTS=" 47 | ports: 48 | - 3000:3000 49 | " 50 | ONDEMAND_PORTS=" 51 | ports: 52 | - 8081:80 53 | " 54 | XDMOD_PORTS=" 55 | ports: 56 | - 8082:80 57 | " 58 | KEYCLOAK_PORTS=" 59 | ports: 60 | - 8083:8080 61 | " 62 | else 63 | ES_PORTS= 64 | KIBANA_PORTS= 65 | PROXY_PORTS= 66 | GRAFANA_PORTS= 67 | XDMOD_PORTS= 68 | KEYCLOAK_PORTS= 69 | fi 70 | 71 | SUBNET=${SUBNET:-"10.11"} 72 | SUBNET6=${SUBNET6:-"2001:db8:1:1::"} 73 | NODELIST=${NODELIST:-"scaleout/nodelist"} 74 | 75 | if [ ! -z "$SLURM_BENCHMARK" ] 76 | then 77 | MYSQL_VOLUMES=" 78 | volumes: 79 | - type: tmpfs 80 | target: /var/lib/mysql 81 | " 82 | NODES_COUNT=100 83 | BUILD_ARGS=" 84 | SLURM_BENCHMARK: ${SLURM_BENCHMARK} 85 | " 86 | else 87 | MYSQL_VOLUMES="" 88 | NODES_COUNT=9 89 | BUILD_ARGS="" 90 | fi 91 | 92 | if [ ! -s "$NODELIST" -o "$SLURM_BENCHMARK" ] 93 | then 94 | if [ ! -z "$FEDERATION" ] 95 | then 96 | c_sub=5 97 | [ -f "$NODELIST" ] && unlink "$NODELIST" 2>&1 >/dev/null 98 | for c in $FEDERATION 99 | do 100 | #generate list nodes per cluster 101 | seq 0 $NODES_COUNT | while read i 102 | do 103 | echo "$(printf "$c-node%02d" $i) $c ${SUBNET}.${c_sub}.$((${i} + 10)) ${SUBNET6}${c_sub}:$((${i} + 10))" 104 | done >> $NODELIST 105 | 106 | c_sub=$((c_sub+1)) 107 | done 108 | else 109 | #generate list of 10 nodes 110 | seq 0 $NODES_COUNT | while read i 111 | do 112 | echo "$(printf "node%02d" $i) cluster ${SUBNET}.5.$((${i} + 10)) ${SUBNET6}5:$((${i} + 10))" 113 | done > $NODELIST 114 | fi 115 | fi 116 | 117 | unlink scaleout/hosts.nodes 118 | cat "$NODELIST" | while read name cluster ip4 ip6 119 | do 120 | [ ! -z "$ip4" ] && echo "$ip4 $name" >> scaleout/hosts.nodes 121 | [ ! -z "$ip6" ] && echo "$ip6 $name" >> scaleout/hosts.nodes 122 | done 123 | 124 | HOSTLIST=" extra_hosts: 125 | - \"db:${SUBNET}.1.3\" 126 | - \"db:${SUBNET6}1:3\" 127 | - \"slurmdbd:${SUBNET}.1.2\" 128 | - \"slurmdbd:${SUBNET6}1:2\" 129 | - \"login:${SUBNET}.1.5\" 130 | - \"login:${SUBNET6}1:5\" 131 | - \"rest:${SUBNET}.1.6\" 132 | - \"rest:${SUBNET6}1:6\" 133 | - \"proxy:${SUBNET}.1.7\" 134 | - \"proxy:${SUBNET6}1:7\" 135 | - \"es01:${SUBNET}.1.15\" 136 | - \"es01:${SUBNET6}1:15\" 137 | - \"es02:${SUBNET}.1.16\" 138 | - \"es02:${SUBNET6}1:16\" 139 | - \"es03:${SUBNET}.1.17\" 140 | - \"es03:${SUBNET6}1:17\" 141 | - \"kibana:${SUBNET}.1.18\" 142 | - \"kibana:${SUBNET6}1:18\" 143 | - \"influxdb:${SUBNET}.1.19\" 144 | - \"influxdb:${SUBNET6}1:19\" 145 | - \"grafana:${SUBNET}.1.20\" 146 | - \"grafana:${SUBNET6}1:20\" 147 | - \"open-ondemand:${SUBNET}.1.21\" 148 | - \"open-ondemand:${SUBNET6}1:21\" 149 | - \"xdmod:${SUBNET}.1.22\" 150 | - \"xdmod:${SUBNET6}1:22\" 151 | - \"keycloak:${SUBNET}.1.23\" 152 | - \"keycloak:${SUBNET6}1:23\" 153 | " 154 | 155 | if [ ! -z "$FEDERATION" ] 156 | then 157 | FIRST_CLUSTER="$(echo "$FEDERATION" | awk '{print $1}')" 158 | FIRST_MGMTNODE="${FIRST_CLUSTER}-mgmtnode" 159 | c_sub=5 160 | 161 | for c in $FEDERATION 162 | do 163 | HOSTLIST="${HOSTLIST} - \"${c}-mgmtnode:${SUBNET}.${c_sub}.1\""$'\n' 164 | HOSTLIST="${HOSTLIST} - \"${c}-mgmtnode:${SUBNET6}${c_sub}:1\""$'\n' 165 | HOSTLIST="${HOSTLIST} - \"${c}-mgmtnode2:${SUBNET}.${c_sub}.2\""$'\n' 166 | HOSTLIST="${HOSTLIST} - \"${c}-mgmtnode2:${SUBNET6}${c_sub}:2\""$'\n' 167 | 168 | c_sub=$((c_sub + 1)) 169 | done 170 | else 171 | FIRST_CLUSTER="cluster" 172 | FIRST_MGMTNODE="mgmtnode" 173 | HOSTLIST="${HOSTLIST} - \"mgmtnode:${SUBNET}.1.1\""$'\n' 174 | HOSTLIST="${HOSTLIST} - \"mgmtnode:${SUBNET6}1:1\""$'\n' 175 | HOSTLIST="${HOSTLIST} - \"mgmtnode2:${SUBNET}.1.4\""$'\n' 176 | HOSTLIST="${HOSTLIST} - \"mgmtnode2:${SUBNET6}1:4\""$'\n' 177 | fi 178 | 179 | LOGGING=" 180 | tty: true 181 | logging: 182 | driver: local 183 | cap_add: 184 | - SYS_PTRACE 185 | - SYS_ADMIN 186 | - MKNOD 187 | - SYS_NICE 188 | - SYS_RESOURCE 189 | security_opt: 190 | - seccomp:unconfined 191 | - apparmor:unconfined 192 | " 193 | 194 | XDMOD=" 195 | xdmod: 196 | build: 197 | context: ./xdmod 198 | network: host 199 | image: xdmod:latest 200 | environment: 201 | - SUBNET=\"${SUBNET}\" 202 | - SUBNET6=\"${SUBNET6}\" 203 | - container=docker 204 | hostname: xdmod 205 | command: [\"/sbin/startup.sh\"] 206 | networks: 207 | internal: 208 | ipv4_address: ${SUBNET}.1.22 209 | ipv6_address: ${SUBNET6}1:22 210 | volumes: 211 | $SYSDFSMOUNTS 212 | - xdmod:/xdmod/ 213 | $XDMOD_PORTS 214 | $LOGGING 215 | $HOSTLIST 216 | " 217 | 218 | if [ "$DISABLE_XDMOD" ] 219 | then 220 | XDMOD="" 221 | fi 222 | 223 | if [ "$CLOUD" ] 224 | then 225 | CLOUD_MOUNTS=" 226 | - type: bind 227 | source: $(readlink -e $(pwd)/cloud_socket) 228 | target: /run/cloud_socket 229 | " 230 | else 231 | CLOUD_MOUNTS="" 232 | fi 233 | # disable Linux specific options 234 | [ $MAC ] && LOGGING= 235 | 236 | cat < requested hostname 18 | dcompose = sys.argv[1] 19 | 20 | # Make sure the socket does not already exist 21 | try: 22 | os.unlink(server_address) 23 | except OSError: 24 | if os.path.exists(server_address): 25 | raise 26 | 27 | # Create a UDS socket 28 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 29 | sock.bind(server_address) 30 | #allow anyone to write to the socket 31 | os.chmod(server_address, stat.S_IROTH | stat.S_IWOTH) 32 | 33 | # Listen for incoming connections 34 | sock.listen(1) 35 | 36 | os.system("%s up --remove-orphans --build --scale cloud=%s --no-recreate -d" % (dcompose, active_nodes)) 37 | 38 | while True: 39 | connection=None 40 | try: 41 | print('waiting for a connection', file=sys.stderr) 42 | connection, client_address = sock.accept() 43 | print('new connection', file=sys.stderr) 44 | 45 | connection.settimeout(10) 46 | data = connection.recv(4096).decode('utf-8').strip() 47 | connection.shutdown(socket.SHUT_RD) 48 | print('received "%s"' % (data), file=sys.stderr) 49 | if data: 50 | op = data.split(":", 1) 51 | if op[0] == "stop": 52 | tag=node_names[op[1]] 53 | 54 | os.system("docker rm -f \"%s\"" % (quote(tag))) 55 | node_names.pop(tag, None) 56 | connection.sendall(b'ACK') 57 | active_nodes -= 1 58 | elif op[0] == "start": 59 | #increase node count by 1 60 | requested_nodes.add(op[1]) 61 | active_nodes += 1 62 | os.system("%s up --scale cloud=%s --no-recreate -d" % (dcompose, active_nodes)) 63 | connection.sendall(b'ACK') 64 | elif op[0] == "whoami": 65 | found=False 66 | 67 | # already known hash 68 | for requested_node, short_node in node_names.items(): 69 | if short_node == op[1]: 70 | found=True 71 | break 72 | 73 | if not found: 74 | short_node=op[1] 75 | requested_node = requested_nodes.pop() 76 | node_names[requested_node]=short_node 77 | 78 | if requested_node: 79 | print("responding: %s=%s" % (requested_node, short_node), file=sys.stderr) 80 | connection.sendall(requested_node.encode('utf-8')) 81 | else: 82 | connection.sendall(b'FAIL') 83 | 84 | print("Active Nodes=%s Known Nodes[%s]=%s" % (active_nodes, len(node_names), node_names), file=sys.stderr) 85 | else: 86 | connection.sendall(b'FAIL') 87 | 88 | connection.close() 89 | except socket.timeout: 90 | print('connection timeout', file=sys.stderr) 91 | except BrokenPipeError: 92 | print('ignoring broken pipe', file=sys.stderr) 93 | except KeyboardInterrupt: 94 | print('shutting down', file=sys.stderr) 95 | break; 96 | 97 | sock.close() 98 | os.unlink(server_address) 99 | 100 | #stop the containers 101 | os.system("make stop") 102 | -------------------------------------------------------------------------------- /grafana/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM grafana/grafana:latest 2 | USER root:root 3 | RUN apk add wget 4 | COPY grafana.ini /etc/grafana/grafana.ini 5 | COPY influxdb.yaml /etc/grafana/provisioning/datasources/influxdb.yaml 6 | COPY dashboards.yaml /etc/grafana/provisioning/dashboards/dashboards.yaml 7 | RUN mkdir -p /var/lib/grafana/dashboards 8 | RUN wget 'https://grafana.com/api/dashboards/11057/revisions/1/download' -O /var/lib/grafana/dashboards/11057.json 9 | -------------------------------------------------------------------------------- /grafana/dashboards.yaml: -------------------------------------------------------------------------------- 1 | - name: 'default' 2 | org_id: 1 3 | folder: '' 4 | type: 'file' 5 | options: 6 | folder: '/var/lib/grafana/dashboards' 7 | -------------------------------------------------------------------------------- /grafana/grafana.ini: -------------------------------------------------------------------------------- 1 | ##################### Grafana Configuration Example ##################### 2 | # 3 | # Everything has defaults so you only need to uncomment things you want to 4 | # change 5 | 6 | # possible values : production, development 7 | ;app_mode = production 8 | 9 | # instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty 10 | ;instance_name = ${HOSTNAME} 11 | 12 | #################################### Paths #################################### 13 | [paths] 14 | # Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) 15 | ;data = /var/lib/grafana 16 | 17 | # Temporary files in `data` directory older than given duration will be removed 18 | ;temp_data_lifetime = 24h 19 | 20 | # Directory where grafana can store logs 21 | ;logs = /var/log/grafana 22 | 23 | # Directory where grafana will automatically scan and look for plugins 24 | ;plugins = /var/lib/grafana/plugins 25 | 26 | # folder that contains provisioning config files that grafana will apply on startup and while running. 27 | ;provisioning = conf/provisioning 28 | 29 | #################################### Server #################################### 30 | [server] 31 | # Protocol (http, https, h2, socket) 32 | ;protocol = http 33 | 34 | # The ip address to bind to, empty will bind to all interfaces 35 | ;http_addr = 36 | 37 | # The http port to use 38 | ;http_port = 3000 39 | 40 | # The public facing domain name used to access grafana from a browser 41 | ;domain = localhost 42 | 43 | # Redirect to correct domain if host header does not match domain 44 | # Prevents DNS rebinding attacks 45 | ;enforce_domain = false 46 | 47 | # The full public facing url you use in browser, used for redirects and emails 48 | # If you use reverse proxy and sub path specify full url (with sub path) 49 | ;root_url = %(protocol)s://%(domain)s:%(http_port)s/ 50 | 51 | # Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. 52 | ;serve_from_sub_path = false 53 | 54 | # Log web requests 55 | ;router_logging = false 56 | 57 | # the path relative working path 58 | ;static_root_path = public 59 | 60 | # enable gzip 61 | ;enable_gzip = false 62 | 63 | # https certs & key file 64 | ;cert_file = 65 | ;cert_key = 66 | 67 | # Unix socket path 68 | ;socket = 69 | 70 | # CDN Url 71 | ;cdn_url = 72 | 73 | #################################### Database #################################### 74 | [database] 75 | # You can configure the database connection by specifying type, host, name, user and password 76 | # as separate properties or as on string using the url properties. 77 | 78 | # Either "mysql", "postgres" or "sqlite3", it's your choice 79 | ;type = sqlite3 80 | ;host = 127.0.0.1:3306 81 | ;name = grafana 82 | ;user = root 83 | # If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" 84 | ;password = 85 | 86 | # Use either URL or the previous fields to configure the database 87 | # Example: mysql://user:secret@host:port/database 88 | ;url = 89 | 90 | # For "postgres" only, either "disable", "require" or "verify-full" 91 | ;ssl_mode = disable 92 | 93 | ;ca_cert_path = 94 | ;client_key_path = 95 | ;client_cert_path = 96 | ;server_cert_name = 97 | 98 | # For "sqlite3" only, path relative to data_path setting 99 | ;path = grafana.db 100 | 101 | # Max idle conn setting default is 2 102 | ;max_idle_conn = 2 103 | 104 | # Max conn setting default is 0 (mean not set) 105 | ;max_open_conn = 106 | 107 | # Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) 108 | ;conn_max_lifetime = 14400 109 | 110 | # Set to true to log the sql calls and execution times. 111 | ;log_queries = 112 | 113 | # For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) 114 | ;cache_mode = private 115 | 116 | ################################### Data sources ######################### 117 | [datasources] 118 | # Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. 119 | ;datasource_limit = 5000 120 | 121 | #################################### Cache server ############################# 122 | [remote_cache] 123 | # Either "redis", "memcached" or "database" default is "database" 124 | ;type = database 125 | 126 | # cache connectionstring options 127 | # database: will use Grafana primary database. 128 | # redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. 129 | # memcache: 127.0.0.1:11211 130 | ;connstr = 131 | 132 | #################################### Data proxy ########################### 133 | [dataproxy] 134 | 135 | # This enables data proxy logging, default is false 136 | ;logging = false 137 | 138 | # How long the data proxy waits before timing out, default is 30 seconds. 139 | # This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. 140 | ;timeout = 30 141 | 142 | # How many seconds the data proxy waits before sending a keepalive probe request. 143 | ;keep_alive_seconds = 30 144 | 145 | # How many seconds the data proxy waits for a successful TLS Handshake before timing out. 146 | ;tls_handshake_timeout_seconds = 10 147 | 148 | # How many seconds the data proxy will wait for a server's first response headers after 149 | # fully writing the request headers if the request has an "Expect: 100-continue" 150 | # header. A value of 0 will result in the body being sent immediately, without 151 | # waiting for the server to approve. 152 | ;expect_continue_timeout_seconds = 1 153 | 154 | # The maximum number of idle connections that Grafana will keep alive. 155 | ;max_idle_connections = 100 156 | 157 | # How many seconds the data proxy keeps an idle connection open before timing out. 158 | ;idle_conn_timeout_seconds = 90 159 | 160 | # If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. 161 | ;send_user_header = false 162 | 163 | #################################### Analytics #################################### 164 | [analytics] 165 | # Server reporting, sends usage counters to stats.grafana.org every 24 hours. 166 | # No ip addresses are being tracked, only simple counters to track 167 | # running instances, dashboard and error counts. It is very helpful to us. 168 | # Change this option to false to disable reporting. 169 | ;reporting_enabled = true 170 | 171 | # The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs 172 | ;reporting_distributor = grafana-labs 173 | 174 | # Set to false to disable all checks to https://grafana.net 175 | # for new versions (grafana itself and plugins), check is used 176 | # in some UI views to notify that grafana or plugin update exists 177 | # This option does not cause any auto updates, nor send any information 178 | # only a GET request to http://grafana.com to get latest versions 179 | ;check_for_updates = true 180 | 181 | # Google Analytics universal tracking code, only enabled if you specify an id here 182 | ;google_analytics_ua_id = 183 | 184 | # Google Tag Manager ID, only enabled if you specify an id here 185 | ;google_tag_manager_id = 186 | 187 | #################################### Security #################################### 188 | [security] 189 | # disable creation of admin user on first start of grafana 190 | ;disable_initial_admin_creation = false 191 | 192 | # default admin user, created on startup 193 | ;admin_user = admin 194 | 195 | # default admin password, can be changed before first start of grafana, or in profile settings 196 | ;admin_password = admin 197 | 198 | # used for signing 199 | ;secret_key = SW2YcwTIb9zpOOhoPsMm 200 | 201 | # disable gravatar profile images 202 | ;disable_gravatar = false 203 | 204 | # data source proxy whitelist (ip_or_domain:port separated by spaces) 205 | ;data_source_proxy_whitelist = 206 | 207 | # disable protection against brute force login attempts 208 | ;disable_brute_force_login_protection = false 209 | 210 | # set to true if you host Grafana behind HTTPS. default is false. 211 | ;cookie_secure = false 212 | 213 | # set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" 214 | ;cookie_samesite = lax 215 | 216 | # set to true if you want to allow browsers to render Grafana in a ,