├── .gitignore ├── images ├── cpuUtil.png ├── keypair.png ├── loadAvg.png ├── macros.png ├── power.png ├── retries.png ├── voltage.png ├── airmaxSN.png ├── intervlan.png ├── password.png ├── wanUpload.png ├── wifiXmit.png ├── channelUsage.png ├── totalSwitch.png ├── wanDownload.png └── channelInterference.png ├── .shellcheckrc ├── ssh-run.sh ├── solarpointBattery.sh ├── updatePublicKey.sh ├── README.md └── mca-dump-short.sh /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /images/cpuUtil.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/cpuUtil.png -------------------------------------------------------------------------------- /images/keypair.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/keypair.png -------------------------------------------------------------------------------- /images/loadAvg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/loadAvg.png -------------------------------------------------------------------------------- /images/macros.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/macros.png -------------------------------------------------------------------------------- /images/power.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/power.png -------------------------------------------------------------------------------- /images/retries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/retries.png -------------------------------------------------------------------------------- /images/voltage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/voltage.png -------------------------------------------------------------------------------- /images/airmaxSN.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/airmaxSN.png -------------------------------------------------------------------------------- /images/intervlan.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/intervlan.png -------------------------------------------------------------------------------- /images/password.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/password.png -------------------------------------------------------------------------------- /images/wanUpload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/wanUpload.png -------------------------------------------------------------------------------- /images/wifiXmit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/wifiXmit.png -------------------------------------------------------------------------------- /images/channelUsage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/channelUsage.png -------------------------------------------------------------------------------- /images/totalSwitch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/totalSwitch.png -------------------------------------------------------------------------------- /images/wanDownload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/wanDownload.png -------------------------------------------------------------------------------- /images/channelInterference.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/patricegautier/unifiZabbix/HEAD/images/channelInterference.png -------------------------------------------------------------------------------- /.shellcheckrc: -------------------------------------------------------------------------------- 1 | #-- SC2317 (info): Command appears to be unreachable. Check usage (or ignore if invoked indirectly). 2 | disable=SC2317 3 | -------------------------------------------------------------------------------- /ssh-run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # set -x 3 | set -uo pipefail 4 | 5 | usage() { 6 | cat << EOF 7 | Usage ${0} -i privateKeyPath -p -u user -v -d targetDevice 8 | -i specify private public key pair path 9 | -p specify password file path to be passed to sshpass -f. Note if both -i and -p are provided, the password file will be used 10 | -u SSH user name for Unifi device 11 | -d IP or FQDN for Unifi device 12 | -v verbose and non compressed output 13 | -r explicitly allow RSA (for legacy devices, i.e Airmaxes) 14 | -n empty option, to provide a default for zabbix key expansion 15 | EOF 16 | exit 2 17 | } 18 | 19 | declare SSHPASS_OPTIONS= 20 | declare PRIVKEY_OPTION= 21 | declare PASSWORD_FILE_PATH= 22 | declare SSH_OPTIONS="-o LogLevel=Error -o StrictHostKeyChecking=accept-new" 23 | 24 | while getopts 'i:u:hd:vp:rn' OPT 25 | do 26 | case $OPT in 27 | i) PRIVKEY_OPTION="-i "${OPTARG} ;; 28 | u) USER=${OPTARG} ;; 29 | d) TARGET_DEVICE=${OPTARG} ;; 30 | v) VERBOSE="-v" ;; 31 | p) PASSWORD_FILE_PATH=${OPTARG} ;; 32 | r) SSH_OPTIONS+=" -o PubkeyAcceptedKeyTypes=+ssh-rsa -o HostKeyAlgorithms=+ssh-rsa" ;; 33 | n) true ;; 34 | *) usage ;; 35 | esac 36 | done 37 | 38 | shift $((OPTIND-1)) 39 | declare -a REMOTE_COMMAND=( "$@ ") 40 | 41 | # {$UNIFI_SSHPASS_PASSWORD_PATH} means the macro didn't resolve in Zabbix 42 | if [[ -n "${PASSWORD_FILE_PATH}" ]] && [[ "${PASSWORD_FILE_PATH}" != "{\$UNIFI_SSHPASS_PASSWORD_PATH}" ]]; then 43 | SSHPASS_OPTIONS="-f ${PASSWORD_FILE_PATH} ${VERBOSE:-}" 44 | PRIVKEY_OPTION= 45 | fi 46 | 47 | if [[ -n "${VERBOSE:-}" ]]; then 48 | # shellcheck disable=SC2086 49 | echo ssh ${SSH_OPTIONS} ${PRIVKEY_OPTION} "${USER}@${TARGET_DEVICE}" "${REMOTE_COMMAND[@]}" 50 | fi 51 | 52 | if [[ -n "${SSHPASS_OPTIONS}" ]]; then 53 | # shellcheck disable=SC2086 54 | sshpass "${SSHPASS_OPTIONS}" ssh ${SSH_OPTIONS} ${PRIVKEY_OPTION} "${USER}@${TARGET_DEVICE}" "${REMOTE_COMMAND[@]}" 55 | else 56 | # shellcheck disable=SC2086,SC2029 57 | ssh ${SSH_OPTIONS} ${PRIVKEY_OPTION} "${USER}@${TARGET_DEVICE}" "${REMOTE_COMMAND[@]}" 58 | fi 59 | -------------------------------------------------------------------------------- /solarpointBattery.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # This script will retrieve battery information from a SunMax SolarPoint device in JSON format 4 | # 5 | # With thanks to @pfanntec in the UI community who got the JSON RPC access rolling 6 | # 7 | set -uo pipefail 8 | 9 | usage() { 10 | echo "Usage ${0} -u userName -p password [-v] targetDevice" 11 | echo "-u user name for Solarpoint device" 12 | echo "-p password for Solarpoint device" 13 | echo "-h help" 14 | exit 2 15 | } 16 | 17 | declare TARGET 18 | declare PASSWORD 19 | declare VERBOSE= 20 | declare USERNAME 21 | 22 | while getopts 'u:p:vh' OPT 23 | do 24 | case "$OPT" in 25 | u) USERNAME=${OPTARG} ;; 26 | p) PASSWORD=${OPTARG} ;; 27 | v) VERBOSE=true ;; 28 | *) usage ;; 29 | esac 30 | done 31 | 32 | shift $((OPTIND-1)) 33 | TARGET=$* 34 | 35 | if [ -z "${TARGET:-}" ]; then 36 | echo "Missing Target device" 37 | usage; 38 | fi 39 | 40 | if [ -z "${USERNAME:-}" ]; then 41 | echo "Missing user name" 42 | usage; 43 | fi 44 | 45 | if [ -z "${PASSWORD:-}" ]; then 46 | echo "Missing password" 47 | usage; 48 | fi 49 | 50 | if [[ -n "${VERBOSE}" ]]; then set -x; fi 51 | 52 | SessionJSON=$(curl -s "https://$TARGET/ubus"\ 53 | --insecure \ 54 | -H "Content-Type: application/json" \ 55 | -H "Accept: application/json"\ 56 | --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"call\",\"params\":[\"00000000000000000000000000000000\",\"session\",\"login\",{\"username\":\"$USERNAME\",\"password\":\"$PASSWORD\",\"timeout\":30}]}") 57 | 58 | if (( $? )); then 59 | echo "Failed to connect to https://$TARGET/ubus" 60 | exit 1 61 | fi 62 | 63 | if [[ -n "${VERBOSE}" ]]; then echo "${SessionJSON}"; fi 64 | 65 | SESSION_ID=$(echo "${SessionJSON}" | jq ".result[1].ubus_rpc_session") 66 | 67 | if [[ -n "${VERBOSE}" ]]; then echo SessionID="${SESSION_ID}"; fi 68 | 69 | if [[ -z "${SESSION_ID}" ]]; then 70 | echo "Could not retrieve Session ID: ${SessionJSON}" 71 | exit 1 72 | else 73 | BatteryData=$(curl -s "https://$TARGET/ubus"\ 74 | --insecure\ 75 | -H "Content-Type: application/json"\ 76 | -H "Accept: application/json"\ 77 | --data "{\"jsonrpc\":\"2.0\",\"id\":344,\"method\":\"call\",\"params\":[$SESSION_ID,\"battery\",\"stats\",{\"timeout\":30}]}") 78 | echo "${BatteryData}" 79 | fi 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /updatePublicKey.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # set -x 3 | 4 | usage() 5 | { 6 | cat << EOF 7 | Usage "${0}" [-d] [-R] [-i privateKeyPath] [-s ] [-u user] user@targetMachine 8 | -i specify private public key pair 9 | -s use sshpass with given password file. Will ask for the password if sshpass is not installed 10 | -d disable strict host key checking with SSH option StrictHostKeyChecking=no 11 | -b for dropbear (used in protect cameras) additionally copy ~/.ssh/authorized_keys to /var/etc/dropbear 12 | -u run the command as 13 | -R remove the local authorized key for that host 14 | -4 force ipv4 15 | -n no ping - do not preflight with ping 16 | -B bind interface for ssh 17 | EOF 18 | exit 2 19 | } 20 | 21 | STRICT="" 22 | unset DROPBEAR 23 | unset SUDO_USER 24 | unset IPV4 25 | unset REMOVE_KEY 26 | unset TARGET 27 | unset PRIVKEY_PATH 28 | unset SSH_PASS_FILE 29 | 30 | while getopts 'i:s:dbu:4RhnB:' OPT 31 | do 32 | case $OPT in 33 | i) PRIVKEY_PATH=${OPTARG} ;; 34 | s) SSH_PASS_FILE=${OPTARG} ;; 35 | d) STRICT="-o StrictHostKeyChecking=no" ;; 36 | b) DROPBEAR=true ;; 37 | u) SUDO_USER="sudo -u "${OPTARG} ;; 38 | 4) IPV4="-4" ;; 39 | R) REMOVE_KEY=true;; 40 | n) NOPING=true;; 41 | B) BIND_INTERFACE_OPTION="-B ${OPTARG}" ;; 42 | *) usage ;; 43 | esac 44 | done 45 | 46 | shift $((OPTIND-1)) 47 | 48 | 49 | TARGET=$* 50 | 51 | 52 | if [[ -z ${TARGET} ]]; then 53 | usage 54 | fi 55 | 56 | 57 | if [[ -z ${PRIVKEY_PATH} ]]; then 58 | PRIVKEY_PATH=${HOME}"/.ssh/id_rsa" 59 | fi 60 | 61 | PUBKEY_PATH=${PRIVKEY_PATH}".pub" 62 | PUBKEY=$(<"${PUBKEY_PATH}") || exit 1; 63 | 64 | if [[ -z "${PUBKEY}" ]]; then 65 | echo "Could not read public key at ${PUBKEY_PATH}" 66 | exit 1 67 | fi 68 | 69 | HOST=$(echo "${TARGET}" | awk -F'@' '{print $2}') 70 | 71 | if [[ -z "${HOST}" ]]; then 72 | echo "Could not parse host from entry: ${TARGET}" 73 | echo "expected user@host" 74 | exit 1 75 | fi 76 | 77 | if [[ -n "${REMOVE_KEY}" ]]; then 78 | ssh-keygen -R "${HOST}" || exit 1 79 | fi 80 | 81 | if [[ -z "${NOPING}" ]]; then 82 | #echo "HOST = $HOST" 83 | #set -x 84 | # Quick check to see if it is pingable 85 | trap - SIGALRM 86 | ping -c 1 -t 5 "${HOST}" >> /dev/null 87 | R=$? 88 | if ! [[ $R -eq 0 ]]; then 89 | echo "${HOST} is not reachable - ping returned $R" 90 | exit 1 91 | fi 92 | fi 93 | 94 | #shellcheck disable=SC2086 95 | ${SUDO_USER} ssh ${BIND_INTERFACE_OPTION} ${IPV4} -i "${PRIVKEY_PATH}" -q -o "BatchMode yes" "${TARGET}" true 96 | PUBKEY_OK=$? 97 | SSHPASS=$(command -v sshpass) 98 | 99 | if [ ${PUBKEY_OK} != '0' ]; then 100 | echo "Need to update public key for ${TARGET}" 101 | unset DROPBEAR_CMD 102 | if [[ -n "${DROPBEAR}" ]]; then 103 | DROPBEAR_CMD=" && cp .ssh/authorized_keys /var/etc/dropbear/" 104 | fi 105 | if [[ -z ${SSH_PASS_FILE} ]] || ! [[ -e ${SSH_PASS_FILE} ]] || [[ -z ${SSHPASS} ]]; then 106 | #shellcheck disable=SC2086 107 | ${SUDO_USER} ssh ${BIND_INTERFACE_OPTION} ${IPV4} ${STRICT} "${TARGET}" "mkdir -p .ssh && echo '${PUBKEY}' >> .ssh/authorized_keys ${DROPBEAR_CMD}" || exit 1; 108 | else 109 | #shellcheck disable=SC2086 110 | ${SUDO_USER} sshpass -f "${SSH_PASS_FILE}" ssh ${IPV4} ${BIND_INTERFACE_OPTION} ${STRICT} "${TARGET}" "mkdir -p .ssh && echo '${PUBKEY}' >> .ssh/authorized_keys ${DROPBEAR_CMD}" || exit 1; 111 | fi 112 | fi 113 | 114 | #ssh-copy-id ${TARGET} > /dev/null 115 | 116 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # unifiZabbix 2 | 3 | This projet contains a collection of Templates to monitor Unifi and other UBNT devices with Zabbix: APs, Switches, Routers (USG and UDMP), AirMax devices, and NVRs 4 | 5 | I am currently running those on the current versions of the base software as of Oct 2022: Zabbix 6.2.x, a mix of Unifi 4.x, 5.x and 6.x APs and switches, AirMax 8.7.1, UDMP 1.12.x, controller 7.2.x 6 | 7 | 8 | # Setup 9 | 10 | ## Zabbix 6.2 11 | 12 | I am now testing and exporting from zabbix server 6.4. I am not sure how far backwards compatible those templates are with older versions of Zabbix. 13 | 14 | It may be problematic to import those templates in anything less than 6.2. A couple of people have asked to have a version based on 6.0. For now, look at the workaround in https://github.com/patricegautier/unifiZabbix/issues/64 15 | 16 | 17 | ## Install jq and expect on your Zabbix server 18 | 19 | You need to install jq on your system: https://stedolan.github.io/jq/ 20 | 21 | On Raspbian, this can be done with: 22 | 23 | sudo apt-get install jq 24 | 25 | You also need to install expect, again on raspbian that can be done with 26 | 27 | sudo apt-get install expect 28 | 29 | ## Install mca-dump-short and ssh-run scripts as a Zabbix external script 30 | 31 | You need to install mca-dump-short.sh and ssh-run in Zabbix's external script directory 32 | 33 | Please confirm where that directory is from the variable ExternalScripts in your zabbix server conf at /etc/zabbix/zabbix_server.conf. On my system this is set to: 34 | 35 | ExternalScripts=/usr/lib/zabbix/externalscripts 36 | 37 | After cp-ing the scripta to that directory, make sure you have the permissions necessary for zabbix to execute this script: 38 | 39 | chown zabbix:zabbix /usr/lib/zabbix/externalscripts /usr/lib/zabbix/externalscripts/mca-dump-short.sh 40 | chmod a+x /usr/lib/zabbix/externalscripts /usr/lib/zabbix/externalscripts/mca-dump-short.sh 41 | 42 | and the same for ssh-run 43 | 44 | ## Import the Unifi templates into Zabbix 45 | 46 | Import zbx_export_templates.yaml into Zabbix, from Configuration > Templates > Import 47 | 48 | You should now have the following templates available, and it should be pretty self explanatory what type of device you need to link them to in Zabbix. 49 | 50 | Unifi AP 51 | Unifi Switch 52 | Unifi USG 53 | Unifi UDMP 54 | UBNT AirMax 55 | Unifi WiFi Site 56 | Unifi Protect Cloud Key 57 | Unifi Protect NVR4 58 | SunMax SolarPoint 59 | Unifi SSH Host 60 | Unsupported Item 61 | 62 | 63 | • You will need to assign the templates with the matching type to hosts you have to create in Zabbix for your unifi devices. Use the 'Agent' interface in Zabbix with the proper IP or DNS entry. 64 | 65 | A couple of things on top of that: 66 | 67 | • Unifi SSH Host: You have to assign that one to all your devices, on top of the template for that specific type. 68 | 69 | For example a Switch should have 2 templates assigned to it: Unifi SSH Host and Unifi Switch 70 | 71 | UPDATE: It is no longer necessary to assign SSH Host to APs, Switches, UDMPs or USGs; The corresponding templates are now pulling those values directly. The number of SSH operations and general load on devices is basically halved with this change, since AP and Switches are the most common devices. 72 | You should still assign this template to NVRs and Cloud Keys. A future version will remove that requirement as well. 73 | 74 | • Unifi SSH High Priority Host is a variant of the first template with higher alert levels that can be used instead; don't assign both to the same device 75 | 76 | • Unifi Wifi Site is meant to aggregate WiFi traffic across your wifi networks for a Unifi site. Just assign it to one of the APs that can see all the networks in question and assign the {$AP_GROUP} macro for that host to the name of a zabbix host group that contains all the APs for that site. 77 | 78 | ## Setup SSH from your Zabbix server to your Unifi devices via public/private keypair 79 | 80 | These templates use public key SSH to access APs, Switches, Routers, AirMax stations and retrieve data directly, using the mca-dump or mca-status command line utility. Your zabbix server (or your proxies if you use those) will need public key SSH access to all the unifi devices they are monitoring: 81 | 82 | 1/ You should generate a new key pair for this. 83 | 84 | The templates are set up to work with a public-private key pair. For a primer on that you can check out for ex https://www.redhat.com/sysadmin/passwordless-ssh. Since you need zabbix to be able to use those without you in front of the keyboard, you need an empty passphrase. 85 | 86 | Zabbix is finicky and this is the specific way I needed to run the generation get a workable keypair (no passphrase, pem format). From your Zabbix server, run: 87 | 88 | sudo mkdir ~/.ssh/zabbix && sudo chown zabbix ~/.ssh/zabbix && cd ~/.ssh/zabbix 89 | sudo -u zabbix ssh-keygen -P "" -t rsa -m pem -f zb_id_rsa 90 | 91 | This is what I end up with: 92 | 93 | pi@pi:~/.ssh/zabbix $ ls -l 94 | total 20 95 | -rw------- 1 zabbix zabbix 1675 Jul 23 18:57 zb_id_rsa 96 | -rw-r--r-- 1 zabbix zabbix 391 Jul 23 18:57 zb_id_rsa.pub 97 | drwxr-xr-x 2 zabbix zabbix 4096 Sep 13 2020 . 98 | drwxr-xr-x 3 pi pi 4096 Mar 1 13:54 .. 99 | 100 | 101 | 2/ You will need to specifically enable SSH access on the unifi devices. 102 | 103 | For most devices, there is one setting in the Unifi controller UI in Settings > Site and one for the UDMP in the UDMP advanced settings which is separate. This is where you specify the username and password that you will use to log in via SSH 104 | 105 | 3/ You then need to send your public key to all the devices you want to monitor 106 | 107 | For managed devices (APs, Switches), the Unifi controller has handy UI to install your public key on all the devices. This is by far the easiest way to do this. 108 | 109 | On network app v9, go to Preferences (gear icon bottom left) > System > Advanced > Device Authentication 110 | 111 | and that will let you add SSH keys that get uploaded to all the known devices. 112 | 113 | for UDMPs and AirMax devices, you will need to do it by hand. *ssh-copy-id* helps there, esp. on the UDMP since those will embarrasingly wipe all your keys at every firmware update and reboot (seriously UBNT): 114 | 115 | sudo -u zabbix ssh-copy-id -i yourUserName@oneOfYourUnifiDevicesIP 116 | 117 | I have a more sophisticated script I use to do this at https://github.com/patricegautier/certRenewalScripts/blob/master/updatePublicKey.sh 118 | 119 | IMPORTANT NOTE: on some Unifi devices (APs and Switches in particular) the authorized keys are stored not in the usual ~/.ssh/authorized_keys, but in ./var/etc/dropbear/authorized_keys. If you provision those keys from the controller UI or using the updatePublicKey script above it will hit the right spot, but ssh-copy-id will not 120 | 121 | 122 | 3/ So now check that your zabbix server can actually get in with SSH with: 123 | 124 | sudo -u zabbix ssh -i yourUserName@oneOfYourUnifiDevicesIP 125 | 126 | If you are set up correctly that should get you in *without asking for a password* 127 | 128 | You can also check that the script used to retrieve data is working correctly for a given device with: 129 | 130 | sudo -u zabbix /usr/lib/zabbix/externalscripts/mca-dump-short.sh -d -u -i -t 131 | 132 | You should get a JSON document in return. 133 | 134 | If you use a Zabbix proxy, it will initiate the connection to the hosts it monitors, so you need to run those tests there. 135 | 136 | 137 | 4/ You then need to point Zabbix to those keys. 138 | 139 | In your Zabbix conf file (/etc/zabbix/zabbix_server.conf typically) add: 140 | 141 | SSHKeyLocation=/the/path/to/your/keys 142 | 143 | in my case I have: 144 | 145 | SSHKeyLocation=/home/pi/.ssh/zabbix 146 | 147 | ## Zabbix Proxies 148 | 149 | If some of your unifi devices are monitored through a proxy you will need to: 150 | 151 | 1/ Install jq/expect and mca-dump-short on the proxy host as well, as detailed in the first 2 steps above 152 | 2/ make sure the proxy host can get to the devices it monitors via SSH as explained above 153 | 154 | ## Running the Zabbix server in a container 155 | 156 | if you run your Zabbix server in a Docker container (as I do), you have to keep in mind that all these SSH accesses take place from within that container. 157 | 158 | A few things then: 159 | 160 | 1/ You need to make sure your private keys are going to resist the container coming and going, and set them up in a persistent docker volume: 161 | 162 | from my Zabbix Server docker-compose.yml: 163 | 164 | volumes: 165 | - ${HOME}/Deployment/zabbixServer/sshKeys:/var/lib/zabbix/ssh_keys:ro 166 | - ${HOME}/Deployment/zabbixServer/externalScripts:/usr/lib/zabbix/externalscripts:ro 167 | ... 168 | 169 | note that /var/lib/zabbix/ssh_keys is the default location for zabbix keys, and so the running container will find them there. 170 | 171 | Also note that I have /usr/lib/zabbix/externalscripts also mapped to a persistent volume; you can just pust mca_dump_short.sh and others in that volume 172 | 173 | 2/ You can then either run all the ssh commands to setup/manage keys from within the Zabbix server container or from the outside targetting the persistent volume. From inside you might run something like: 174 | 175 | docker exec zabbix-server ssh-keygen -P "" -t rsa -m pem -f /var/lib/zabbix/ssh_keys/zb_id_rsa 176 | 177 | to create the public/private key pair. Similarly to explicitly check the container can get to a particular device: 178 | 179 | docker exec zabbix-server /usr/lib/zabbix/externalscripts/mca-dump-short.sh -d -u -i /var/lib/zabbix/ssh_keys/zb_id_rsa -t 180 | 181 | 182 | 183 | ## Macros 184 | 185 | In Zabbix, in Administration > General > Macros, you will need to set a value for __*all*__ the following macros: 186 | 187 | ### {$UNIFI_USER} 188 | The username that will let the zabbix server (or proxy) log in to your unifi devices via SSH 189 | 190 | ### {$UNIFI_SSH_PRIV_KEY_PATH} 191 | The full path where to find the public private key pair to be able to SSH into your Unifi devices. The private key should be in the SSHKeyLocation directory from your zabbix conf file. For my system for ex, this is set to /home/pi/.ssh/zabbix/zb_id_rsa 192 | 193 | ### {$UNIFI_SSH_PORT} 194 | Set this macro on each host, in case you are using a non standard port for SSH (!=22) 195 | 196 | ### {$UNIFI_PRIV_KEY} 197 | The file name for your private key in SSHKeyLocation. For me this is set to zb_id_rsa 198 | 199 | ### {$UNIFI_PUB_KEY} 200 | The file name for your public key in SSHKeyLocation. For me this is set to zb_id_rsa.pub 201 | 202 | ### {$UNIFI_SSHPASS_PASSWORD_PATH} 203 | If you are having trouble geting ssh going with public/private key pair authentication, you can optionally supply the path of a file that contains the SSH password to your Unifi devices. If supplied, the template will use sshpass to provide the password to ssh. There are more security implications to doing this than using the keypair method.. 204 | 205 | ### {$UNIFI_CHECK_FREQUENCY} 206 | How often to poll Unifi devices for new data. I have this set to '1m' 207 | 208 | ### {$UNIFI_CHECK_TIMEOUT} 209 | How long to wait for devices to return data. I have this set to '5' (not 5s), as some switch regularly take 2-3s to respond. Note that you should have the overall Zabbix TIMEOUT, or ZBX_TIMEOUT if you are using the container version set to at a value greater than this 210 | 211 | ### {$UNIFI_DISCOVERY_FREQUENCY} 212 | How often to discover features of devices, mostly switch port names; I have this set to 15mns 213 | 214 | ### {$UNIFI_ALERT_TEMP} 215 | The temperature in Celsius above which to alert. I have set this to '90'. 216 | 217 | ### {$UNIFI_ALERT_PERIOD} 218 | The period after which to alert for most checks. I have this set to '10m'. The triggers on this period are level 'Warning' 219 | 220 | ### {$UNIFI_ALERT_LONG_PERIOD} 221 | The period after which to alert for lower priority checks, unsupported items for example. I have this set to '12h'. The triggers on this period are level 'Not classified' 222 | 223 | ### {$PROTECT_CAMERA_PASSWORD} 224 | Set this to your cameras' password. There's UI in the protect controller to set this on all cameras at once. 225 | 226 | You will also need to enable SSH for cameras, the instructions are at: 227 | 228 | https://help.ui.com/hc/en-us/articles/360015877853-UniFi-Protect-Enabling-Camera-SSH-Access 229 | 230 | 231 | ### {$PROTECT_LOW_BANDWIDTH} 232 | The threshold of camera outoing bandwidth below which to alert. I have this set to 500000 i.e 500kbps. 233 | 234 | ### {$UNIFI_SMOOTHING_PERIOD} 235 | I have defined some moving average items with the suffix _smooth to help make graphs easier to read. This is set to '10m' for me. 236 | 237 | ### {$UNIFI_LOAD_AVERAGE_MEDIUM} 238 | The load average value above which to issue a info. The consensus is 1 for this. Note that for switches and APs this value has less meaning since they process packets with specialized HW and this macro is overridden in the template to avoid too many warnings 239 | 240 | ### {$UNIFI_LOAD_AVERAGE_HIGH} 241 | The load average value above which to issue a warning. I have this set to 2. Note that for switches and APs this value has less meaning since they process packets with specialized HW and this macro is overridden in the template to avoid too many warnings 242 | 243 | ### {$UNIFI_CHANNEL_INTERFERENCE_INFO_THRESHOLD} 244 | ### {$UNIFI_CHANNEL_INTERFERENCE_AVERAGE_THRESHOLD} 245 | The percentages above which to issue and info/average event for channel interferences. I have this set to 30 and 50 respectively 246 | You can customize this for 2G/5G with {$UNIFI_CHANNEL_INTERFERENCE_INFO_THRESHOLD:"ng"} or {$UNIFI_CHANNEL_INTERFERENCE_AVERAGE_THRESHOLD:"na"} values 247 | 248 | 249 | ### {$UNIFI_CHANNEL_USAGE_INFO_THRESHOLD} 250 | ### {$UNIFI_CHANNEL_USAGE_AVERAGE_THRESHOLD} 251 | The percentages above which to issue and info/average event for Channel usage. I have this set to 30 and 50 respectively 252 | You can customize this for 2G/5G with {$UNIFI_CHANNEL_USAGE_INFO_THRESHOLD:"ng"} or {$UNIFI_USAGE_INTERFERENCE_INFO_THRESHOLD:"na"} values 253 | 254 | ### {$UNIFI_PORT_USAGE_INFO_THRESHOLD} 255 | ### {$UNIFI_PORT_USAGE_WARNING_THRESHOLD} 256 | ### {$UNIFI_PORT_USAGE_AVERAGE_THRESHOLD} 257 | The percentages above which to issue and info/average event for switch Port usage. I have this set to 40, 60 and 50 respectively 258 | 259 | Note: those macros do not come with the import, you have to create them by hand; they will survive subsequent imports.. 260 | 261 | 262 | 263 | # SUCCESS! 264 | 265 | If you got this far, congratulations the install is complete! Now for the funner part: 266 | 267 | 268 | # Graphs 269 | 270 | If everything is working you should be able to see data flowing in the Monitoring > Latest Data section of Zabbix. Time to set up some graphs.. I have some basic graphs setup in the templates themselves, unfortunately Zabbix doesn't to have a good way to share the fancier graphs as it does for templates, so here is a quick rundown of what I have setup..: 271 | 272 | 273 | ## UDMP|USG Wan Download 274 | ![Wan Download](/images/wanDownload.png) 275 | 276 | ## UDMP|USG Wan Upload 277 | ![Wan Upload](/images/wanUpload.png) 278 | 279 | Observe the tight correlation between upload bandwidth and latency.. Cable technology at its finest! 280 | 281 | ## Router: InterVLAN traffic by network 282 | ![InterVLAN](/images/intervlan.png) 283 | 284 | I have found this one to be useful to get router usage down 285 | 286 | ## Total Switch Traffic by Switch 287 | ![Switch Traffic](/images/totalSwitch.png) 288 | 289 | Useful to point to potential bottlenecks 290 | 291 | ## WiFi Channel Usage by AP 292 | ![Channel Interference](/images/channelUsage.png) 293 | 294 | ## WiFi Channel Interference bu AP 295 | ![Channel Interference](/images/channelInterference.png) 296 | 297 | ## WiFi Transmission by Network 298 | ![Retries](/images/wifiXmit.png) 299 | 300 | ## WiFi Retries 301 | ![Retries](/images/retries.png) 302 | 303 | ## Airmax S/N and Airtime by AirMax device 304 | ![Airmad](/images/airmaxSN.png) 305 | 306 | 307 | # SunMax Solarpoint Support 308 | 309 | ## Install solarpointBattery.sh 310 | 311 | Following the same steps as mca-dump-short.sh above, install solarpointBattery.sh as a Zabbix external script 312 | 313 | ## Macros 314 | 315 | The SunMax SolarPoints do not support SSH, so you'll have to set up the following macros seperately, either in Administration > General > Macros or on the host directly depending on your case: 316 | 317 | ### {$SOLARPOINT_USERNAME} 318 | The username that will let the zabbix server (or proxy) log in to the SolarPoint device 319 | 320 | ### {$SOLARPOINT_PASSWORD} 321 | The password that will let the zabbix server (or proxy) log in to the SolarPoint device 322 | 323 | That will give you access to power production and consumption, as well as set a trigger on PoE ports being suspended 324 | 325 | ## Some Graph Examples 326 | 327 | ## Power Production and Consumption 328 | ![Wan Download](/images/power.png) 329 | 330 | ## Battery Voltage 331 | ![Wan Download](/images/voltage.png) 332 | 333 | 334 | 335 | # Updating to a new version 336 | 337 | • You should simply have to import the updates zbx_export_templates.yaml into zabbix and those will replace the current template. 338 | • Also check if there are any updates to any of the scripts (mcaDumpShort.sh in particular) and if so please copy those files in the right spot on your zabbix server/proxies 339 | • Macros that you have defined in Administration > General > Macros will survive importing a new template version, so no action needed there 340 | • Macros that you have defined directly on one of your devices that uses one of the templates in this project also survive 341 | • If you did define some macros on the templates themselves, those will get blown out during the import.. You can either move those macros to one of the aforementioned spts or if that's not practicaly you can also define your own template that links to one of the templates in this projects, put the macros on that new template and have your devices use that. It's also a good way to customize the templates to your case, though I would of course hope you would contribute your enhancements back! 342 | 343 | 344 | # Troubleshooting - Notes 345 | 346 | • Your zabbix server log file (/var/log/zabbix/zabbix\_server.log usually) can be a good source of debugging info, esp if you set DebugLevel=4 in /etc/zabbix/zabbix_server.conf. Restart the Zabbix server with 347 | 348 | sudo service zabbix-server restart 349 | 350 | • mcaDumpShort.sh is also logging all errors to /tmp/mcaDumpShort.err on the zabbixServer (or proxy if you are using one). It's a good source of info to debug issues too. 351 | 352 | • If you see timeouts, there are 2 values to experiment with: 353 | 354 | in your zabbix server conf, usually /etc/zabbix/zabbix_server.conf add adjust the zabbix timeout (default is 3s): 355 | 356 | TimeOut=30 357 | 358 | and then in the Zabbix UI, change the macro value for UNIFI_CHECK_TIMEOUT in Administration > General Macros so sth a little smaller than the first value, maybe 25 in this case. 359 | 360 | • Macros Cheat Sheet 361 | 362 | This is my set of values 363 | 364 | ![Macros](/images/macros.png) 365 | 366 | • libSSH can be a source of problems: 367 | 368 | - if some of your items randomly fail with 'Cannot read data from SSH server' (in the UI or in /var/log/zabbix/zabbix_server.log), the likely culprit is an outdated version of libssh, which sometimes returns an error code even on success. You have to compile the last version from sources from libssh.org and recompile I'm afraid.. This was a problem on Raspbian buster for libssh 0.8.x and is confirmed fixed with libssh 0.9.5 at least. Note 03/23: This seems to be resolved with Zabbix 6.4.. 369 | 370 | On Raspbian Bullseye you can find libssl at https://packages.debian.org/bullseye/amd64/libssh-4/download 371 | 372 | I am just downloading sources from https://www.libssh.org and recompiling. 373 | 374 | Incidentally, this is still a problem with Zabbix 6.0.0 containers, which are still packaged with libssh 0.9.3 375 | 376 | - In Ubuntu 20.04 the "ssh.run" key does not work with the standard libssh. Just install the latest version from https://launchpad.net/~kedazo/+archive/ubuntu/libssh-0.7.x and everything is OK. 377 | 378 | 379 | • if an import fails with 'Invalid parameter "/interfaceid": cannot be empty.', it might be caused by the presence of an older version of the templates. Remove them before re-importing.. 380 | 381 | • SSH to Unifi devices is invoked with the SSH option "-o StrictHostKeyChecking=accept-new" which means it will automatically accept their SSH host key on first connection to that IP or Host Name. The default SSH setting is to ask for the user's confirmation on first connection but I deemed the extra convenience of not having to do this to be worth it in the context of a Home/Small Business Unifi setup 382 | 383 | • The reason for the existence of the mca-dump-short external script instead of using SSH items directly if you are wondering is that there is a 64k limit to ssh.run items in zabbix, which we run into running mca-dump on large switches for ex. To raise that limit in Zabbix you have to recompile from sources and I didn't want to have that dependency. The only reliable way I found around this is to run an external script to retrieve the mca-dump data from device, and then post-process it with jq to make the data < 64k on the way into Zabbix 384 | 385 | 386 | # Future Additions 387 | 388 | ## Auto-discovery of devices based on controller connection 389 | 390 | i.e automatically create all the proper hosts connected to the proper templates via a single connection to the Unifi controller 391 | 392 | ## Better SSH debugging 393 | 394 | Most of the pain in setting those templates up is debugging the SSH connections.. Add pre-processing to check for valid json on mca-dump-short to all templates 395 | 396 | 397 | -------------------------------------------------------------------------------- /mca-dump-short.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -uo pipefail 3 | 4 | declare HE_RSA_SSH_KEY_OPTIONS='-o PubkeyAcceptedKeyTypes=+ssh-rsa -o HostKeyAlgorithms=+ssh-rsa' 5 | 6 | #AP|SWITCH|SWITCH_FEATURE_DISCOVERY|SWITCH_DISCOVERY|UDMP|USG 7 | declare -A VALIDATOR_BY_TYPE 8 | VALIDATOR_BY_TYPE["AP"]=".vap_table? != null and .radio_table != null" 9 | VALIDATOR_BY_TYPE["UDMP"]=".network_table? != null" 10 | VALIDATOR_BY_TYPE["USG-LITE"]=".network_table? != null" 11 | VALIDATOR_BY_TYPE["USG"]="( .network_table? != null ) and ( .network_table | map(select(.mac!=null)) | length>0 )" 12 | declare -A OPTIONAL_VALIDATOR_BY_TYPE 13 | declare -A OPTION_MESSAGE 14 | #OPTIONAL_VALIDATOR_BY_TYPE["USG"]=" ( ( .[\"system-stats\"].temps | length ) == 4 ) " 15 | #OPTION_MESSAGE["USG"]="missingTemperatures" 16 | 17 | declare RETRIABLE_ERROR=250 18 | declare SSH_CONNECT_TIMEOUT=5 19 | 20 | #--------------------------------------------------------------------------------------- 21 | # Utilities 22 | 23 | function runWithTimeout () { 24 | local timeout=$1 25 | shift 26 | "$@" & 27 | local child=$! 28 | # Avoid default notification in non-interactive shell for SIGTERM 29 | trap -- "" SIGTERM 30 | local now; now=$(date +%s%N); now="${now:0:-6}" 31 | local endDate; endDate=$(( now + timeout*1000 )) 32 | local running=true 33 | ( while (( now < endDate )) && [[ -n "${running}" ]]; 34 | do 35 | sleep 0.1 36 | if ! ps -p ${child} > /dev/null; then 37 | running= 38 | fi 39 | now=$(date +%s%N); now="${now:0:-6}" 40 | done 41 | if [[ -n "${running}" ]]; then kill ${child} 2> /dev/null; fi 42 | ) & 43 | wait ${child} 44 | } 45 | 46 | 47 | function errorJsonWithReason() { 48 | local reason; reason=$(echo "$1" | tr -d "\"'\n\r" ) 49 | local t; t=$(date +"%T") 50 | echo '{ "at":"'"${t}"'", "r":"'"${reason}"'", "device":"'"${TARGET_DEVICE}"'", "mcaDumpError":"Error" }' 51 | } 52 | 53 | function validationErrorJsonWithReason() { 54 | local reason; reason=$(echo "$1" | tr -d "\"'\n\r" ) 55 | local t; t=$(date +"%T") 56 | echo '{ "at":"'"${t}"'", "r":"'"${reason}"'", "device":"'"${TARGET_DEVICE}"'", "mcaDumpValidationError":"Error" }' 57 | } 58 | 59 | function timeoutJsonWithReason() { 60 | local reason; reason=$(echo "$1" | tr -d "\"'\n\r" ) 61 | local t; t=$(date +"%T") 62 | echo '{ "at":"'"${t}"'", "r":"'"${reason}"'", "device":"'"${TARGET_DEVICE}"'", "mcaDumpTimeout":"Error" }' 63 | } 64 | 65 | 66 | function insertWarningIntoJsonOutput() { 67 | local warning=$1 68 | local output=$2 69 | echo "${output}" | jq ". + { mcaDumpWarning: { \"${warning}\": true } }" 70 | echoErr "warning: $warning" 71 | } 72 | 73 | function echoErr() { 74 | local error=$1 75 | { 76 | echo "----------------------------------" 77 | echo "$(date) $TARGET_DEVICE" 78 | echo "Command: ${FULL_ARGS}" 79 | echo 80 | echo " ${error}" 81 | } >> "${errFile}" 82 | if [[ -f "/./.dockerenv" ]]; then # also echo the error to docker logs if running inside a container 83 | { 84 | echo " ${error}" 85 | } >> /proc/1/fd/1 86 | fi 87 | } 88 | 89 | function issueSSHCommand() { 90 | local command=$* 91 | if [[ -n "${VERBOSE:-}" ]]; then 92 | #shellcheck disable=SC2086 93 | echo ${SSHPASS_OPTIONS} ssh ${SSH_PORT} ${VERBOSE_SSH} ${HE_RSA_SSH_KEY_OPTIONS} ${BATCH_MODE} -o LogLevel=Error -o ConnectTimeout=${SSH_CONNECT_TIMEOUT} -o StrictHostKeyChecking=accept-new ${PRIVKEY_OPTION} "${USER}@${TARGET_DEVICE}" "$command" 94 | fi 95 | #shellcheck disable=SC2086 96 | ${SSHPASS_OPTIONS} ssh ${SSH_PORT} ${VERBOSE_SSH} ${HE_RSA_SSH_KEY_OPTIONS} ${BATCH_MODE} -o LogLevel=Error -o ConnectTimeout=${SSH_CONNECT_TIMEOUT} -o StrictHostKeyChecking=accept-new ${PRIVKEY_OPTION} "${USER}@${TARGET_DEVICE}" "$command" 97 | } 98 | 99 | declare TRUNCATE_SIZE=1000000 # 1M 100 | declare TRUNCATE_FREQUENCY=86400 #1D 101 | function truncateFileOnceADay() { 102 | local file=$1 103 | if [[ -f "$file" ]]; then 104 | local size 105 | if ! size=$(wc -c < "$file"); then return; fi 106 | if (( size > TRUNCATE_SIZE )); then 107 | local haveToTrunc=1 108 | local truncMarker="$file.truncMarker" 109 | if [[ -f "$truncMarker" ]]; then 110 | local trunkMarkerDate; 111 | if ! trunkMarkerDate=$(date -r "$truncMarker" +%s); then return; fi 112 | local now; now=$(date +%s) 113 | if (( now - trunkMarkerDate < TRUNCATE_FREQUENCY )); then 114 | haveToTrunc=0 115 | fi 116 | fi 117 | if (( haveToTrunc )); then 118 | local tmpFile="$file.tmpTrunc" 119 | tail -c "$TRUNCATE_SIZE" "$file" > "$tmpFile" 120 | mv "$tmpFile" "$file" 121 | touch "$truncMarker" 122 | fi 123 | fi 124 | fi 125 | } 126 | 127 | #--------------------------------------------------------------------------------------- 128 | # Fan Discovery 129 | 130 | function fanDiscovery() { 131 | local -n exitCode=$1 132 | exitCode=0 133 | shift 134 | local sensors; sensors=$(issueSSHCommand sensors | grep -E "^fan[0-9]:" | cut -d' ' -f1) 135 | exitCode=$? 136 | if (( exitCode == 0 )); then 137 | local first=true 138 | echo -n "[ " 139 | for fan in $sensors; do 140 | if [[ -n "$fan" ]]; then 141 | if [[ -z "$first" ]]; then echo -n ","; else first=; fi 142 | echo -n "{ \"name\": \"${fan::-1}\" }" 143 | fi 144 | done 145 | echo -n " ]" 146 | fi 147 | } 148 | 149 | #--------------------------------------------------------------------------------------- 150 | # Switch Discovery 151 | 152 | 153 | # thanks @zpolisensky for this contribution 154 | #shellcheck disable=SC2016 155 | PORT_NAMES_AWK=' 156 | BEGIN { IFS=" \n"; first=1; countedPortId=0 } 157 | match($0, "^interface 0/[0-9]+$") { 158 | portId=substr($2,3) 159 | } 160 | match($0, "^interface [A-z0-9]+$") { 161 | countedPortId=countedPortId+1 162 | portId=countedPortId 163 | } 164 | /description / { 165 | desc="" 166 | defaultDesc="Port " portId 167 | for (i=2; i<=NF; i++) { 168 | f=$i 169 | if (i==2) f=substr(f,2) 170 | if (i==NF) 171 | f=substr(f,1,length(f)-1) 172 | else 173 | f=f " " 174 | desc=desc f 175 | } 176 | if (first != 1) printf "| " 177 | first=0 178 | if ( desc == defaultDesc) 179 | desc="-" 180 | else 181 | desc="(" desc ")" 182 | printf ".port_table[" portId-1 "] += { \"port_desc\": \"" desc "\" }" 183 | }' 184 | 185 | 186 | 187 | 188 | 189 | declare SWITCH_DISCOVERY_DIR="/tmp/unifiSwitchDiscovery" 190 | function startSwitchDiscovery() { 191 | local jqProgram=$1 192 | local exp; exp=$(command -v expect) 193 | if [[ -z "${exp}" ]]; then exp=$(ls /usr/bin/expect); fi 194 | if [[ -z "${exp}" ]]; then 195 | OUTPUT=$(errorJsonWithReason "please install 'expect' to run SWITCH_DISCOVERY") 196 | return 1 197 | else 198 | mkdir -p "${SWITCH_DISCOVERY_DIR}" 199 | #shellcheck disable=SC2034 200 | # o=$(runWithTimeout 60 retrievePortNamesInto "${jqProgram}") & 201 | # nohup needs a cmd-line utility 202 | # nohup runWithTimeout 60 retrievePortNamesInto "${jqProgram}" & 203 | #(set -m; runWithTimeout 60 retrievePortNamesInto "${jqProgram}" &) & 204 | #runWithTimeout 60 retrievePortNamesInto "${jqProgram}" & 205 | runWithTimeout 60 retrievePortNamesInto "${jqProgram}" > /dev/null 2> /dev/null < /dev/null & disown 206 | fi 207 | return 0 208 | } 209 | 210 | 211 | function retrievePortNamesInto() { 212 | local jqFile=$1 213 | local logFile="$1-$RANDOM.log" 214 | local outStream="/dev/null" 215 | local options= 216 | #sleep $(( TIMEOUT + 1 )) # This ensures we leave the switch alone while mca-dump proper is processed; the next invocation will find the result 217 | if [[ -n "${VERBOSE:-}" ]]; then 218 | #shellcheck disable=SC2086 219 | echo ${SSHPASS_OPTIONS} spawn ssh ${SSH_PORT} ${VERBOSE_SSH} ${HE_RSA_SSH_KEY_OPTIONS} -o LogLevel=Error -o ConnectTimeout=${SSH_CONNECT_TIMEOUT} -o LogLevel=Error -o StrictHostKeyChecking=accept-new "${PRIVKEY_OPTION}" "${USER}@${TARGET_DEVICE}" >&2 220 | fi 221 | if [[ -n "${VERBOSE_PORT_DISCOVERY:-}" ]]; then 222 | options="-d" 223 | outStream="/dev/stdout" 224 | fi 225 | 226 | #shellcheck disable=SC2086 227 | /usr/bin/expect ${options} > "${outStream}" <" { 276 | send -- "enable\r" 277 | expect "(UBNT) #" 278 | 279 | send -- "terminal length 0\r" 280 | expect "(UBNT) #" 281 | 282 | send -- "show run\r" 283 | log_file -noappend ${logFile}; 284 | 285 | expect "(UBNT) #" 286 | send -- "exit\r" 287 | 288 | } 289 | "telnet: not found\r\n" { 290 | send -- "cli\r" 291 | expect -re ".*#" 292 | 293 | send -- "terminal length 0\r" 294 | expect -re ".*#" 295 | 296 | send -- "show run\r" 297 | log_file -noappend ${logFile}; 298 | expect -re ".*#" 299 | 300 | send "exit\r" 301 | } 302 | } 303 | } 304 | EOD 305 | local exitCode=$? 306 | if (( exitCode )); then 307 | { echo "$(date) $TARGET_DEVICE"; 308 | echo " retrievePortNamesInto failed with code $exitCode"; 309 | echo "Full command was $FULL_ARGS" 310 | if [[ -f "$logFile" ]]; then 311 | cat "$logFile" 312 | fi 313 | } >> "${errFile}" 314 | exit "${exitCode}" 315 | fi 316 | 317 | if [[ -s "$logFile" ]]; then 318 | #shellcheck disable=SC2002 319 | local tmpFile="$1-$RANDOM.tmp" 320 | cat "$logFile" | tr -d '\r' | awk "$PORT_NAMES_AWK" > "${tmpFile}" 321 | if [[ -s "$tmpFile" ]]; then 322 | mv "${tmpFile}" "${jqFile}" 323 | else 324 | echoErr "Empty port names file" 325 | fi 326 | rm -f "$logFile" "$tmpFile" 2>/dev/null 327 | else 328 | echoErr "** No Show Run output" 329 | fi 330 | 331 | } 332 | 333 | function insertPortNamesIntoJson() { 334 | local -n out=$1 335 | local jqProgramFile=$2 336 | local json=$3 337 | if [[ -s "${jqProgramFile}" ]]; then 338 | if [[ -n "${VERBOSE:-}" ]]; then 339 | echo "jqProgramFile: " 340 | cat "${jqProgramFile}" 341 | echo; echo 342 | fi 343 | #shellcheck disable=SC2034 344 | out=$(echo "${json}" | jq -f "${jqProgramFile}" -r) 345 | #rm "$jqProgramFile" 2>/dev/null # we now leave it for the next guy 346 | else 347 | echoErr "Empty jqProgramFile" 348 | #shellcheck disable=SC2034 349 | out=$json 350 | fi 351 | } 352 | 353 | #--------------------------------------------------------------------------------------------------------------------- 354 | # mca-dump invocation 355 | 356 | 357 | function invokeUpToNTimesWithDelay() { 358 | local count=$1 359 | local delay=$2 360 | shift 2 361 | local returnCode=0 362 | local invocations 363 | for (( invocations=0; invocations < count; invocations++ )); do 364 | "$@" 365 | returnCode=$? 366 | if (( returnCode==0 || returnCode != RETRIABLE_ERROR )); then 367 | invocations=$count 368 | else 369 | echoErr " Warning: Retrying $1 request" 370 | sleep "$delay" 371 | fi 372 | done 373 | return $returnCode 374 | } 375 | 376 | function invokeMcaDump() { 377 | local deviceType=$1 378 | local jqProgram=$2 379 | local -n exitCode=$3; exitCode=0 380 | local -n output=$4; output= 381 | local -n jsonOutput=$5; jsonOutput= 382 | 383 | local indentOption="--indent 0" 384 | 385 | 386 | local delay=1 # the CPU is very wimpy on the USG-lite, ssh into it affects the usage. Sleeping 2s gets a better CPU read 387 | case "${deviceType:-}" in 388 | 389 | AP) JQ_OPTIONS='del (.port_table) | 390 | del(.radio_table[]?.scan_table) | del(.scan_radio_table) | 391 | del(.radio_table[]?.spectrum_table) | 392 | ( .vap_table[]|= ( .clientCount = ( .sta_table|length ) ) ) | del (.vap_table[]?.sta_table)' ;; 393 | SWITCH | SWITCH_DISCOVERY) JQ_OPTIONS='del (.port_table[]?.mac_table?)' ;; 394 | SWITCH_FEATURE_DISCOVERY) JQ_OPTIONS="[ { power: .port_table | any ( .poe_power >= 0 ) ,\ 395 | total_power_consumed_key_name: \"total_power_consumed\",\ 396 | max_power_key_name: \"max_power\",\ 397 | max_power: .total_max_power,\ 398 | percent_power_consumed_key_name: \"percent_power_consumed\",\ 399 | has_eth1: .has_eth1,\ 400 | has_temperature: .has_temperature,\ 401 | temperature_key_name: \"temperature\",\ 402 | overheating_key_name: \"overheating\",\ 403 | has_fan: .has_fan,\ 404 | fan_level_key_name: \"fan_level\" 405 | } ]" ;; 406 | UDMP| USG) JQ_OPTIONS='del (.dpi_stats) | del(.fingerprints) | del( .network_table[]? | select ( .address == null ))' ;; 407 | USG-LITE) JQ_OPTIONS='del (.dpi_stats) | del(.fingerprints) | del( .network_table[]? | select ( .address == null ))' 408 | delay=4 ;; # the CPU is very wimpy on the USG-lite, ssh into it affects the usage. Sleeping 2s gets a better CPU read 409 | *) echo "Unknown device Type: '${DEVICE_TYPE:-}'"; usage ;; 410 | esac 411 | 412 | local errorFile="/tmp/mca-dump-short-$RANDOM$RANDOM.err" 413 | #shellcheck disable=SC2086 414 | output=$(timeout --signal=HUP --kill-after=5 "${TIMEOUT}"\ 415 | ${SSHPASS_OPTIONS} ssh ${SSH_PORT} ${VERBOSE_SSH} ${HE_RSA_SSH_KEY_OPTIONS} ${BATCH_MODE} -o LogLevel=Error -o ConnectTimeout=${SSH_CONNECT_TIMEOUT} -o StrictHostKeyChecking=accept-new ${PRIVKEY_OPTION} "${USER}@${TARGET_DEVICE}"\ 416 | ${delay:+sleep ${delay}\;} mca-dump 2>&1) 417 | exitCode=$? 418 | #shellcheck disable=SC2034 419 | jsonOutput="${output}" 420 | 421 | if [[ -n "${VERBOSE:-}" ]]; then 422 | echoErr "mcaDump.sh output" 423 | echoErr "${jsonOutput}" 424 | fi 425 | 426 | if (( exitCode == 124 )); then 427 | output=$(timeoutJsonWithReason "timeout ($exitCode)") 428 | elif (( exitCode )) || [[ -z "${output}" ]]; then 429 | output=$(errorJsonWithReason "$(echo "Remote pb: "; echo "${output}" )") 430 | exitCode=1 431 | else 432 | if [[ -n "${JQ_VALIDATOR:-}" ]]; then 433 | local validation; validation=$(echo "${output}" | jq "${JQ_VALIDATOR}") 434 | exitCode=$? 435 | if [[ -z "${validation}" || "${validation}" == "false" ]] || (( exitCode )); then 436 | output=$(validationErrorJsonWithReason "validationError: ${JQ_VALIDATOR}") 437 | exitCode=$RETRIABLE_ERROR 438 | fi 439 | fi 440 | if (( ! exitCode )) && [[ -n "${JQ_OPTION_VALIDATOR:-}" ]]; then 441 | local optionValidation; optionValidation=$(echo "${output}" | jq "${JQ_OPTION_VALIDATOR}") 442 | exitCode=$? 443 | if [[ -z "${optionValidation}" ]] || [[ "${optionValidation}" == "false" ]] || (( exitCode != 0 )); then 444 | local message=${OPTION_MESSAGE["${DEVICE_TYPE}"]:-"unknownWarning"} 445 | output=$(insertWarningIntoJsonOutput "$message" "$output") 446 | fi 447 | fi 448 | if (( ! exitCode )); then 449 | local errorFile="/tmp/jq$RANDOM$RANDOM.err" 450 | local jqInput=${output} 451 | output= 452 | #shellcheck disable=SC2086 453 | output=$(echo "${jqInput}" | jq ${indentOption} "${JQ_OPTIONS}" 2> "${errorFile}") 454 | exitCode=$? 455 | if (( exitCode )) || [[ -z "${output}" ]]; then 456 | local errorMessage; errorMessage="jq ${indentOption} ${JQ_OPTIONS} returned status code $exitCode 457 | $(cat "$errorFile")" 458 | if [[ -n "${VERBOSE:-}" ]]; then errorMessage="${errorMessage} 459 | Input was: ${jqInput}" 460 | fi 461 | output=$(errorJsonWithReason "${errorMessage}") 462 | exitCode=1 463 | fi 464 | rm -f "${errorFile}" 2>/dev/null 465 | fi 466 | fi 467 | 468 | if (( ! exitCode )) && [[ "${DEVICE_TYPE:-}" == 'SWITCH_DISCOVERY' ]]; then 469 | # do not wait anymore for retrievePortNamesInto 470 | # this will ensure we don't time out, but sometimes we will use an older file 471 | # wait 472 | errorFile="/tmp/jq${RANDOM}${RANDOM}.err" 473 | local jqInput="${output}" 474 | output= 475 | insertPortNamesIntoJson output "${jqProgram}" "${jqInput}" 2> "${errorFile}" 476 | local code=$? 477 | if (( code )) || [[ -z "${output}" ]]; then 478 | output=$(errorJsonWithReason "insertPortNamesIntoJson failed with error code $code; $(cat "$errorFile")") 479 | exitCode=1 480 | fi 481 | rm "${errorFile}" 2>/dev/null 482 | fi 483 | return "$exitCode" 484 | } 485 | 486 | 487 | #------------------------------------------------------------------------------------------------ 488 | 489 | 490 | function usage() { 491 | 492 | local error="${1:-}" 493 | if [[ -n "${error}" ]]; then 494 | echo "${error}" 495 | echo 496 | fi 497 | 498 | cat <<- EOF 499 | Usage ${0} -i privateKeyPath -p -u user -v -d targetDevice [-t AP|SWITCH|SWITCH_FEATURE_DISCOVERY|SWITCH_DISCOVERY|UDMP|UDMP_FAN_DISCOVERY|UDMP_TEMP_DISCOVERY|USG|USG-LITE] 500 | -i specify private public key pair path 501 | -p specify password file path to be passed to sshpass -f. Note if both -i and -p are provided, the password file will be used 502 | -u SSH user name for Unifi device 503 | -d IP or FQDN for Unifi device 504 | -o alternate port for SSH connection 505 | -t Unifi device type 506 | -v verbose and non compressed output 507 | -w verbose output for port discovery 508 | -x extreme debugging 509 | -o max timeout (3s minimum) 510 | -O echoes debug and timing info to /tmp/mcaDumpShort.log; errors are always echoed to /tmp/mcaDumpShort.err 511 | -V Provide a JQ expression that must return a non empty output to validate the results. A json error is returned otherwise 512 | -b run SSH in batch mode (do not ask for passwords) 513 | EOF 514 | exit 1 515 | } 516 | 517 | function checkOptForMissingMacro() { 518 | local v=$1 519 | local t=$2 520 | if [[ "$v" == "{\$$t}" ]]; then 521 | echo "Please set the {\$$t} macro in zabbix > Administration" 522 | fi 523 | } 524 | 525 | #------------------------------------------------------------------------------------------------ 526 | 527 | declare SSHPASS_OPTIONS= 528 | declare PRIVKEY_OPTION= 529 | declare PASSWORD_FILE_PATH= 530 | declare VERBOSE_OPTION= 531 | declare TIMEOUT=15 532 | declare VERBOSE_SSH= 533 | declare SSH_PORT= 534 | declare TARGET_DEVICE_PORT= 535 | declare logFile="/tmp/mcaDumpShort.log" 536 | declare errFile="/tmp/mcaDumpShort.err" 537 | declare ECHO_OUTPUT= 538 | declare VERBOSE= 539 | declare FULL_ARGS="$0 $*" 540 | declare BATCH_MODE= 541 | 542 | while getopts 'i:u:t:hd:vp:wm:o:OV:U:P:ebx' OPT 543 | do 544 | case $OPT in 545 | i) checkOptForMissingMacro "${OPTARG}" "UNIFI_SSH_PRIV_KEY_PATH}" 546 | PRIVKEY_OPTION="-i "${OPTARG} ;; 547 | u) checkOptForMissingMacro "${OPTARG}" "USER}" 548 | USER=${OPTARG} ;; 549 | t) DEVICE_TYPE=${OPTARG} ;; 550 | d) TARGET_DEVICE=${OPTARG} ;; 551 | P) TARGET_DEVICE_PORT=${OPTARG} ;; 552 | v) export VERBOSE=true ;; 553 | p) PASSWORD_FILE_PATH=${OPTARG} ;; 554 | w) VERBOSE_PORT_DISCOVERY=true ;; 555 | m) logFile=${OPTARG} ;; 556 | o) TIMEOUT=$(( OPTARG-1 )) ;; 557 | O) ECHO_OUTPUT=true ;; 558 | V) JQ_VALIDATOR=${OPTARG} ;; 559 | x) set -x ;; 560 | b) BATCH_MODE="-o BatchMode=yes" ;; 561 | e) echo -n "$(errorJsonWithReason "simulated error")"; exit 1 ;; 562 | U) if [[ -n "${OPTARG}" ]] && [[ "${OPTARG}" != "{\$UNIFI_VERBOSE_SSH}" ]]; then 563 | export VERBOSE_SSH="${OPTARG}" 564 | fi ;; 565 | *) usage ;; 566 | esac 567 | done 568 | 569 | declare EXIT_CODE=0 570 | declare OUTPUT= 571 | declare JSON_OUTPUT= 572 | 573 | 574 | 575 | if [[ -n "${ECHO_OUTPUT:-}" ]]; then 576 | START_TIME=$(date +%s) 577 | fi 578 | 579 | if [[ -n "${VERBOSE:-}" ]]; then 580 | export VERBOSE_OPTION="-v" 581 | fi 582 | 583 | if [[ -z "${TARGET_DEVICE:-}" ]]; then 584 | usage "Please specify a target device with -d" 585 | fi 586 | 587 | if [[ -z "${DEVICE_TYPE:-}" ]]; then 588 | usage "Please specify a device type with -t" 589 | fi 590 | 591 | if [[ "${TARGET_DEVICE_PORT}" == "{\$UNIFI_SSH_PORT}" ]]; then 592 | TARGET_DEVICE_PORT="" 593 | fi 594 | if [[ -n "${TARGET_DEVICE_PORT}" ]]; then 595 | if (( TARGET_DEVICE_PORT == 0 )) || (( TARGET_DEVICE_PORT < 0 )) || (( TARGET_DEVICE_PORT > 65535 )); then 596 | echo "Please specify a valid port with -P ($TARGET_DEVICE_PORT was specified)" >&2 597 | usage 598 | fi 599 | if (( TARGET_DEVICE_PORT != 10050 )); then 600 | SSH_PORT="-p ${TARGET_DEVICE_PORT}" 601 | fi 602 | fi 603 | 604 | 605 | if [[ -z "${USER:-}" ]]; then 606 | echo "Please specify a username with -u" >&2 607 | usage 608 | fi 609 | 610 | 611 | if [[ -z "${JQ_VALIDATOR:-}" ]]; then 612 | JQ_VALIDATOR=${VALIDATOR_BY_TYPE["${DEVICE_TYPE}"]:-} 613 | fi 614 | declare JQ_OPTION_VALIDATOR=${OPTIONAL_VALIDATOR_BY_TYPE["${DEVICE_TYPE}"]:-} 615 | 616 | 617 | # {$UNIFI_SSHPASS_PASSWORD_PATH} means the macro didn't resolve in Zabbix 618 | if [[ -n "${PASSWORD_FILE_PATH}" ]] && [[ "${PASSWORD_FILE_PATH}" != "{\$UNIFI_SSHPASS_PASSWORD_PATH}" ]]; then 619 | if ! [[ -f "${PASSWORD_FILE_PATH}" ]]; then 620 | echo "Password file not found '$PASSWORD_FILE_PATH'" 621 | exit 1 622 | fi 623 | SSHPASS_OPTIONS="sshpass -f ${PASSWORD_FILE_PATH} ${VERBOSE_OPTION}" 624 | PRIVKEY_OPTION= 625 | fi 626 | 627 | declare JQ_PROGRAM="${SWITCH_DISCOVERY_DIR}/switchPorts-${TARGET_DEVICE}.jq" 628 | if [[ ${DEVICE_TYPE:-} == 'SWITCH_DISCOVERY' ]]; then 629 | startSwitchDiscovery "$JQ_PROGRAM" # asynchronously discover port names 630 | EXIT_CODE=$? 631 | fi 632 | 633 | if (( EXIT_CODE == 0 )); then 634 | case "${DEVICE_TYPE}" in 635 | UDMP_FAN_DISCOVERY) fanDiscovery EXIT_CODE OUTPUT JSON_OUTPUT ;; 636 | *) invokeUpToNTimesWithDelay 2 0 invokeMcaDump "$DEVICE_TYPE" "$JQ_PROGRAM" EXIT_CODE OUTPUT JSON_OUTPUT ;; 637 | esac 638 | fi 639 | 640 | 641 | if [[ -n "${ECHO_OUTPUT:-}" ]]; then 642 | END_TIME=$(date +%s) 643 | DURATION=$(( END_TIME - START_TIME )) 644 | echo "$(date): ${TARGET_DEVICE}:${TARGET_DEVICE_PORT:-} ${DEVICE_TYPE} ${JQ_VALIDATOR:-} : ${DURATION}s - $EXIT_CODE" >> "${logFile}" 645 | if [[ -n "${ECHO_OUTPUT:-}" ]]; then 646 | echo -n "${OUTPUT}" >> "${logFile}" 647 | echo >> "${logFile}" 648 | fi 649 | fi 650 | 651 | if (( EXIT_CODE )); then 652 | echoErr "${OUTPUT}" 653 | echoErr "${JSON_OUTPUT}" 654 | fi 655 | 656 | echo "${OUTPUT}" 657 | 658 | truncateFileOnceADay "$errFile" 659 | truncateFileOnceADay "$logFile" 660 | 661 | exit $EXIT_CODE 662 | 663 | 664 | --------------------------------------------------------------------------------