6 |
5 |
6 |
8 | 
9 |
11 | 
12 |
14 | 
15 |
17 | 
18 |
\n\n"
54 |
55 | # run loop with links
56 | for url in urls:
57 |
58 | # create pretty output
59 | graffiti_out = (url_dict[url]["graffiti"] + ' ').ljust(graffiti_max + 2, '>')
60 | points_out = str(url_dict[url]["total_points"]).rjust(points_max)
61 | rank_out = url_dict[url]["rank"]
62 |
63 | print(f"{graffiti_out} {points_out} points, #{rank_out}.")
64 | text = text + f"{graffiti_out} {points_out} points, %23{rank_out}.\n"
65 |
66 | # end the message
67 | text = text + ""
68 | print()
69 |
70 | # send message with telegram bot
71 | telegram = f'https://api.telegram.org/bot{telegram_bot_api}/sendMessage?chat_id={telegram_chat_id}&parse_mode=html&text={text}'
72 | message = requests.get(telegram)
73 |
--------------------------------------------------------------------------------
/nym/README.md:
--------------------------------------------------------------------------------
1 | # status
2 | This system will alert you with telegram about outdated and inactive status. Also it sends you every hour short info about your node status.
3 |
4 | Instruction:
5 |
6 | 1. Create telegram bot via `@BotFather`, customize it and `get bot API token` ([how_to](https://www.siteguarding.com/en/how-to-get-telegram-bot-api-token)).
7 | 2. Create at least 2 groups: `alarm` and `log`. Customize them, add your bot into your chats and `get chats IDs` ([how_to](https://stackoverflow.com/questions/32423837/telegram-bot-how-to-get-a-group-chat-id)).
8 | 3. Connect to your server and create `status` folder in the `$HOME` directory with `mkdir $HOME/status/`.
9 | 4. In this folder you have to create `nym.sh` file with `nano $HOME/status/nym.sh`. You don't have to do any edits on `nym.sh` file, it's ready to use.
10 | > You can find `nym.sh` in this repository.
11 | 5. Also you have to create as many `nym.conf` files with `nano $HOME/status/nym.conf`, as many nodes you have on the current server. Customize your config files.
12 | > You can find `nym.conf.example` in this repository.
13 | 6. Install `jq` and `bc` packages with `sudo apt-get install jq bc -y`.
14 | 7. Run `bash nym.sh` to check your settings. Normal output:
15 |
16 | ```
17 | root@ubuntu:~/status# bash nym.sh
18 |
19 | /// 2022-05-21 20:07:32 ///
20 |
21 | nym-m | cyberomanov
22 |
23 | stake >>>>> 127314.73 | $50473.92.
24 | salary/m >> 367.20 | $145.58.
25 | unpaid >>>> 77.26 | $30.63.
26 |
27 | root@ubuntu:~/status#
28 | ```
29 |
30 | 8. Create `slash.sh` with `nano $HOME/status/slash.sh`, if you don't have one yet. This bash script will divide group of messages.
31 | > You can find `slash.sh.example` in this repository.
32 | 9. Add some rules with `chmod u+x nym.sh slash.sh`.
33 | 10. Edit crontab with `crontab -e`.
34 | > You can find `crontab.example` in this repository.
35 | 11. Check your logs with `cat $HOME/status/nym.log` or `tail $HOME/status/nym.log -f`.
36 |
--------------------------------------------------------------------------------
/nym/crontab.example:
--------------------------------------------------------------------------------
1 | # status
2 | 2,12,22,32,42,52 * * * * bash $HOME/status/nym.sh >> $HOME/status/nym.log 2>&1
3 |
4 | # slash
5 | 0 * * * * $HOME/status/slash.sh
6 |
--------------------------------------------------------------------------------
/nym/nym.conf.example:
--------------------------------------------------------------------------------
1 | # mixnode identity
2 | IDENTITY="7TWEw9qQxsc8w4WhPAX6zjZ8vuNBdtP21zUVN8K26RkD"
3 |
4 | # moniker
5 | MONIKER="cyberomanov"
6 |
7 | # project note
8 | PROJECT="nym-mainnet"
9 |
10 | # if 'last hour uptime' is lower than X > alarm, 0 to disable
11 | UPTIME="80"
12 |
13 | # chat id for alarm messages with notification enabled
14 | CHAT_ID_ALARM="-187"
15 |
16 | # chat id for log messages
17 | CHAT_ID_STATUS="-1703"
18 |
19 | # bot token for send messages
20 | BOT_TOKEN="228322:xxx-xxx_xxx"
21 |
--------------------------------------------------------------------------------
/nym/nym.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | function __Send() {
4 | # print 'TEXT' into 'nym.log' for the sake of history
5 | echo -e "${TEXT}"
6 |
7 | # add new text to the 'MESSAGE', which will be sent as 'log' or 'alarm'
8 | # if 'SEND' == 1, it becomes 'alarm', otherwise it's 'log'
9 | MESSAGE=${MESSAGE}''$(echo "${TEXT}")'\n'
10 | }
11 |
12 | function __NodeStatus() {
13 | MESSAGE="${PROJECT} ⠀|⠀ ${MONIKER}\n\n"
14 | echo -e "${PROJECT} | ${MONIKER}\n"
15 | SEND=0; MINUTE=10
16 |
17 | RESPONSE=$(curl -s ${CURL}${IDENTITY})
18 | if [[ ${RESPONSE} != *"error"* ]]; then
19 | # get uptime
20 | UPTIME_TOTAL=$(curl -s ${CURL}${IDENTITY}"/uptime")
21 | LAST_HOUR=$(echo ${UPTIME_TOTAL} | jq ".last_hour" | tr -d '"')
22 | LAST_DAY=$(echo ${UPTIME_TOTAL} | jq ".last_day" | tr -d '"')
23 |
24 | # if 'last hour uptime' is low > alarm
25 | if (( ${LAST_HOUR} < ${UPTIME} )); then
26 | SEND=1
27 | TEXT="_hour/day > $LAST_HOUR%/$LAST_DAY%."
28 | __Send
29 | fi
30 |
31 | # get info about node status
32 | TOTAL_INFO=$(curl -s ${CURL}${IDENTITY})
33 |
34 | # if node is outdated > alarm
35 | OUTDATED=$(echo ${TOTAL_INFO} | jq ".outdated" | tr -d '"')
36 | if [[ "${OUTDATED}" == "true" ]]; then
37 | SEND=1
38 | VERSION=$(echo ${TOTAL_INFO} | jq ".mixnode.mix_node.version" | tr -d '"')
39 | TEXT="_version >> ${VERSION}.\n_outdated > ${OUTDATED}."
40 | __Send
41 | fi
42 |
43 | # if status is not active > alarm
44 | STATUS=$(echo ${TOTAL_INFO} | jq ".mixnode.status" | tr -d '"')
45 | if [[ "${STATUS}" != "active" ]]; then
46 | SEND=1
47 | TEXT="_status >>> ${STATUS}."
48 | __Send
49 | fi
50 |
51 | # get info about stake
52 | DELEGATION=$(echo $(echo "scale=2;$(echo ${TOTAL_INFO} | jq ".mixnode.total_delegation.amount" | tr -d '"')/${DENOM}" | bc))
53 | SELF_STAKE=$(echo $(echo "scale=2;$(echo ${TOTAL_INFO} | jq ".mixnode.pledge_amount.amount" | tr -d '"')/${DENOM}" | bc))
54 | TOTAL_STAKE=$(echo ${SELF_STAKE} + ${DELEGATION} | bc -l)
55 | TOTAL_STAKE_CUR=$(printf "%.2f" $(echo "scale=2;${TOTAL_STAKE}*${TOKEN_PRICE_CUR}" | bc))
56 |
57 | # get info about rewards
58 | TOTAL_REWARDS=$(curl -s ${CURL}${IDENTITY}"/estimated_reward")
59 |
60 | # get info about estimated rewards
61 | ESTIMATED_REWARDS_H=$(echo $(echo "scale=2;$(echo ${TOTAL_REWARDS} | jq ".estimated_operator_reward" | tr -d '"')/${DENOM}" | bc))
62 | ESTIMATED_REWARDS_M=$(echo ${ESTIMATED_REWARDS_H}*720 | bc -l)
63 | if (( $(bc <<< "${ESTIMATED_REWARDS_H} < 1") )); then ESTIMATED_REWARDS_H="0${ESTIMATED_REWARDS_H}"; fi
64 | if (( $(bc <<< "${ESTIMATED_REWARDS_M} < 1") )); then ESTIMATED_REWARDS_M="0${ESTIMATED_REWARDS_M}"; fi
65 | ESTIMATED_REWARDS_M_CUR=$(printf "%.2f" $(echo "scale=2;${ESTIMATED_REWARDS_M}*${TOKEN_PRICE_CUR}" | bc))
66 |
67 | # get unpaid rewards
68 | DELEGATOR=$(echo ${TOTAL_INFO} | jq ".address" | tr -d '"')
69 | DELEGATOR_CLAIMABLE=$(curl -sk https://mixnet.api.explorers.guru/api/accounts/${DELEGATOR}/balance | jq ".claimable.amount" | tr -d '"')
70 | UNPAID=$(echo $(echo "scale=2;${DELEGATOR_CLAIMABLE}/${DENOM}" | bc))
71 | if (( $(bc <<< "${UNPAID} < 1") )); then UNPAID="0${UNPAID}"; fi
72 | UNPAID_CUR=$(printf "%.2f" $(echo "scale=2;${UNPAID}*${TOKEN_PRICE_CUR}" | bc))
73 |
74 | # find the longest value
75 | MAX=${#UNPAID}
76 | if (( ${#ESTIMATED_REWARDS_M} > ${MAX})); then MAX=${#ESTIMATED_REWARDS_M}; fi
77 | if (( ${#TOTAL_STAKE} > ${MAX})); then MAX=${#TOTAL_STAKE}; fi
78 | MAX=$((MAX+1))
79 |
80 | # pretty output
81 | TOTAL_STAKE_SPACE=$((MAX-${#TOTAL_STAKE})); ESTIMATED_REWARDS_M_SPACE=$((MAX-${#ESTIMATED_REWARDS_M})); UNPAID_SPACE=$((MAX-${#UNPAID}))
82 | TOTAL_STAKE_TEXT=$(printf "stake >>>>>%${TOTAL_STAKE_SPACE}s${TOTAL_STAKE} | ${CURRENCY_SYMB}${TOTAL_STAKE_CUR}.\n")
83 | ESTIMATED_REWARDS_M_TEXT=$(printf "salary/m >>%${ESTIMATED_REWARDS_M_SPACE}s${ESTIMATED_REWARDS_M} | ${CURRENCY_SYMB}${ESTIMATED_REWARDS_M_CUR}.\n")
84 | UNPAID_SPACE_TEXT=$(printf "unpaid >>>>%${UNPAID_SPACE}s${UNPAID} | ${CURRENCY_SYMB}${UNPAID_CUR}.\n")
85 |
86 | TEXT="${TOTAL_STAKE_TEXT}\n${ESTIMATED_REWARDS_M_TEXT}\n${UNPAID_SPACE_TEXT}"
87 | __Send
88 | else
89 | TEXT="_explorer is down."
90 | __Send
91 | fi
92 |
93 | if [[ ${SEND} == "1" ]]; then
94 | curl --header 'Content-Type: application/json' \
95 | --request 'POST' \
96 | --data '{"chat_id":"'"${CHAT_ID_ALARM}"'", "text":"'"$(echo -e "${MESSAGE}")"'", "parse_mode": "html"}' "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
97 | > /dev/null 2>&1
98 | elif (( $(echo "$(date +%M) < ${MINUTE}" | bc -l) )); then
99 | curl --header 'Content-Type: application/json' \
100 | --request 'POST' \
101 | --data '{"chat_id":"'"${CHAT_ID_STATUS}"'", "text":"'"$(echo -e "${MESSAGE}")"'", "parse_mode": "html"}' "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
102 | > /dev/null 2>&1
103 | fi
104 | }
105 |
106 | function Main() {
107 | CURL="https://mixnet.api.explorers.guru/api/mixnodes/"
108 | IP=$(wget -qO- eth0.me)
109 | DENOM="1000000"
110 | TOKEN="nym"
111 | CURRENCY="usd"
112 | CURRENCY_SYMB="$"
113 | TOKEN_PRICE_CUR=$(curl -sk -X GET "https://api.coingecko.com/api/v3/simple/price?ids=${TOKEN}&vs_currencies=${CURRENCY}" -H "accept: application/json" | jq ".nym.usd")
114 |
115 | echo -e " "
116 | echo -e "/// $(date '+%F %T') ///"
117 | echo -e " "
118 |
119 | . $HOME/status/nym.conf
120 | __NodeStatus
121 | }
122 |
123 | # run 'Main'
124 | Main
125 |
--------------------------------------------------------------------------------
/nym/slash.sh.example:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # chat id for log messages, not alarm
4 | CHAT_ID_LOG="-1703"
5 |
6 | # bot token
7 | BOT_TOKEN="228322:xxx-xxx_xxx"
8 |
9 | MESSAGE="/// $(date '+%F %T') ///"
10 |
11 | curl --header 'Content-Type: application/json' \
12 | --request 'POST' \
13 | --data '{"chat_id":"'"$CHAT_ID_LOG"'","text":"'"$(echo -e $MESSAGE)"'", "parse_mode": "html"}' "https://api.telegram.org/bot$BOT_TOKEN/sendMessage"
14 |
--------------------------------------------------------------------------------
/solana/README.md:
--------------------------------------------------------------------------------
1 | https://youtu.be/pbruFZqHkUc
2 |
--------------------------------------------------------------------------------
/tendermint/auto_stake/README.md:
--------------------------------------------------------------------------------
1 | https://youtu.be/pbruFZqHkUc
2 |
--------------------------------------------------------------------------------
/tendermint/node_status/README.md:
--------------------------------------------------------------------------------
1 | # status
2 | This system will alert you with telegram about jails and inactive status. Also it sends you every hour short info about your node status.
3 |
4 | Instruction:
5 |
6 | 1. Create telegram bot via `@BotFather`, customize it and `get bot API token` ([how_to](https://www.siteguarding.com/en/how-to-get-telegram-bot-api-token)).
7 | 2. Create at least 2 groups: `alarm` and `log`. Customize them, add your bot into your chats and `get chats IDs` ([how_to](https://stackoverflow.com/questions/32423837/telegram-bot-how-to-get-a-group-chat-id)).
8 | 3. Connect to your server and create `status` folder in the `$HOME directory` with `mkdir $HOME/status/`.
9 | 4. In this folder, `$HOME/status/`, you have to create `cosmos.sh` file with `nano $HOME/status/cosmos.sh`. You don't have to do any edits on `cosmos.sh` file, it's ready to use.
10 | > You can find `cosmos.sh` in this repository.
11 | 5. In this folder, `$HOME/status/`, you have to create `cosmos.conf` file with `nano $HOME/status/cosmos.conf`. Customize it.
12 | > You can find `cosmos.conf` in this repository.
13 | 6. Also you have to create as many `name.conf` files with `nano $HOME/status/name.conf`, as many nodes you have on the current server. Customize your config files. For ex: I have agoric, gravity and sifchain on the same server, so I have to create 3 files: `agoric.conf`, `gravity.conf` and `sifchain.conf`.
14 | > You can find `name.conf` and `curl.md` in this repository.
15 | 7. Install some packages with `sudo apt-get install jq sysstat bc smartmontools fdisk -y`.
16 | 8. Run `bash cosmos.sh` to check your settings. Normal output:
17 | ```
18 | root@v1131623:~/status# bash cosmos.sh
19 |
20 | /// 2022-07-09 11:42:37 ///
21 |
22 | testnets | load
23 |
24 | cpu >>>>> 68%.
25 | ram >>>>> 47%.
26 | part >>>> 55%.
27 | load >>>> 14.03.
28 |
29 | dws-t | cyberomanov
30 |
31 | exp/me >> 955540/955540.
32 | place >>> 88/200.
33 | stake >>> 34.98 dws.
34 |
35 | root@v1131623:~/status#
36 | ```
37 | 9. Add some rules with `chmod u+x $HOME/status/cosmos.sh`.
38 | 10. Edit crontab with `crontab -e`.
39 | > You can find `crontab` in this repository.
40 | 11. Check your logs with `cat $HOME/status/cosmos.log` or `tail $HOME/status/cosmos.log -f`.
41 |
--------------------------------------------------------------------------------
/tendermint/node_status/cosmos.conf:
--------------------------------------------------------------------------------
1 | # send 'alarm_message' when chain upgrade is in N minutes
2 | # ex: 'UPGRADE_ALARM_IN_MIN=30' will send 'alarm_message' 30, 20 and 10 mins before the chain upgrade
3 | UPGRADE_ALARM_IN_MIN=30
4 |
5 | # timezone from 'https://en.wikipedia.org/wiki/List_of_tz_database_time_zones' > 'TZ database name'
6 | # ex: 'TIMEZONE="Asia/Tbilisi"' is +04:00 UTC, 'TIMEZONE="Africa/Abidjan"' is +00:00 UTC.
7 | TIMEZONE="Africa/Abidjan"
8 |
9 | # server load monitoring: set "false" to disable, set "true" to enable
10 | SEND_LOAD="false"
11 |
12 | # server nickname for 'server load monitoring' message
13 | SERVER="mainnets"
14 |
15 | # cpu/ram/partition threshold in percent for 'alarm_message' trigger
16 | CPU_ALARM=85
17 | RAM_ALARM=85
18 | PARTITION_ALARM=85
19 |
20 | # physical disk percentage used, not partition
21 | DISK_PERCENTAGE_USED_ALARM=80
22 |
23 | # chat_id for alarm messages with enabled notifications
24 | CHAT_ID_ALARM="-187"
25 |
26 | # bot_token for sending messages
27 | BOT_TOKEN="228322:xxx-xxx_xxx"
28 |
--------------------------------------------------------------------------------
/tendermint/node_status/cosmos.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | function __Send() {
4 |
5 | # print 'TEXT' into 'cosmos.log' for the sake of history
6 | echo -e ${TEXT}
7 |
8 | # add new text to the 'MESSAGE', which will be sent as 'log_message' or 'alarm_message'
9 | # if 'SEND' == 1, it becomes 'alarm_message', otherwise it's 'log_message'
10 | MESSAGE=${MESSAGE}''${TEXT}'\n'
11 | }
12 |
13 | function __PreMessage() {
14 |
15 | # init some premessages
16 | CPU_T="cpu_used >>>>"
17 | RAM_T="ram_used >>>>"
18 | SWAP_T="swap_used >>>"
19 | PART_T="part_used >>>"
20 | LOAD_T="serv_load >>>"
21 | DISK_SPARE_T="disk_spare >>"
22 | DISK_USED_T="disk_used >>>"
23 |
24 | MISSED_T="missed >>>>>>"
25 | TRAIN_T="train >>>>>>>"
26 | JAILED_A="_jailed >>>>>"
27 |
28 | GOV_T="gov >>>>>>>>>"
29 |
30 | UPGRADE_T="upgrade >>>>>"
31 | TIME_L_T="_time_left >>"
32 | APPR_T_T="_appr_time >>"
33 |
34 | EXP_ME_T="exp/me >>>>>>"
35 | ACTIVE_A="_active >>>>>"
36 | PLACE_T="place >>>>>>>"
37 | STAKE_T="stake >>>>>>>"
38 |
39 | PRIVKEY_T="priv_key >>>>"
40 | }
41 |
42 | function __DiskVitality() {
43 |
44 | # init some variables
45 | KEY=0
46 |
47 | # trying to install 'smartmontools'
48 | if [[ $(/usr/sbin/smartctl -V 2>&1) == *"not found"* || $(/usr/sbin/fdisk -v 2>&1) == *"not found"* ]]; then
49 | apt-get install smartmontools fdisk -y > /dev/null 2>&1
50 | fi
51 |
52 | # if successfuly installed > check disk
53 | if [[ $(/usr/sbin/smartctl -V 2>&1) != *"not found"* || $(/usr/sbin/fdisk -v 2>&1) != *"not found"* ]]; then
54 | DISK_NAME_STRING=$(/usr/sbin/fdisk -l | grep -e "Disk /dev/*" | grep -oE "/dev/[[:alnum:]]*")
55 | DISK_NAME_ARRAY=($(echo "${DISK_NAME_STRING}" | tr ' ' '\n'))
56 | for i in "${!DISK_NAME_ARRAY[@]}"; do
57 | DISK_INFO=$(/usr/sbin/smartctl -s on -a ${DISK_NAME_ARRAY[i]})
58 | if [[ ${DISK_INFO} != *"Unable to detect device"* ]]; then
59 | KEY=1
60 | DISK_NAME=$(echo ${DISK_NAME_ARRAY[i]})
61 | SPARE=$(echo ${DISK_INFO} | grep -o "Available Spare: [0-9]*" | grep -o "[0-9]*")
62 | SPARE_THRESHOLD=$(echo ${DISK_INFO} | grep -o "Available Spare Threshold: [0-9]*" | grep -o "[0-9]*")
63 | PERCENTAGE_USED=$(echo ${DISK_INFO} | grep -o "Percentage Used: [0-9]*" | grep -o "[0-9]*")
64 |
65 | if [[ $(echo "${SPARE} < ${SPARE_THRESHOLD}" | bc) -eq 1 ]]; then
66 | SEND=1
67 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"\n_${DISK_SPARE_T::-1} ${DISK_NAME:5} has only ${SPARE}% spare."
68 | else
69 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"\n${DISK_SPARE_T} ${DISK_NAME:5} has ${SPARE}% spare."
70 | fi
71 |
72 | if [[ $(echo "${PERCENTAGE_USED} > ${DISK_PERCENTAGE_USED_ALARM}" | bc) -eq 1 ]]; then
73 | SEND=1
74 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"\n_${DISK_USED_T::-1} ${DISK_NAME:5} has ${PERCENTAGE_USED}% used."
75 | else
76 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"\n${DISK_USED_T} ${DISK_NAME:5} has ${PERCENTAGE_USED}% used."
77 | fi
78 | fi
79 | done
80 |
81 | if [[ ${KEY} == 0 ]]; then
82 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"\nthere is no disk which can be tested."
83 | fi
84 | else
85 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"\ninstall tools manually: 'apt-get install smartmontools fdisk -y'."
86 | fi
87 | }
88 |
89 | function __ServerLoad() {
90 |
91 | # add new text to the 'MESSAGE', which will be sent as 'log_message' or 'alarm_message'
92 | MESSAGE="${SERVER} ⠀|⠀ load\n\n"
93 | echo -e "${SERVER} | load\n"
94 |
95 | # init some variables
96 | CPU_ALARM=80
97 | RAM_ALARM=80
98 | PARTITION_ALARM=80
99 | CPU_LOAD_MESSAGE=""
100 | TEXT=""
101 | SEND=0
102 |
103 | # read 'cosmos.conf'
104 | . ./cosmos.conf
105 |
106 | # get CPU load
107 | CPU=$(printf "%.0f" $(echo "scale=2; 100-$(mpstat | tail -1 | awk 'NF {print $NF}')" | bc))
108 | if (( $(echo "${CPU} > ${CPU_ALARM}" | bc -l) )); then
109 | SEND=1
110 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"_${CPU_T::-1} ${CPU}%.\n"
111 | else
112 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"${CPU_T} ${CPU}%.\n"
113 | fi
114 |
115 | # get RAM load
116 | free -g > ~/temp.txt
117 | RAM_TOTAL=$(cat ~/temp.txt | awk '{print $2}' | awk 'NR==2 {print; exit}')"G"
118 | RAM_USED=$(cat ~/temp.txt | awk '{print $3}' | awk 'NR==2 {print; exit}')"G"
119 | RAM_PERC=$(printf "%.0f" $(echo "scale=2; ${RAM_USED}/${RAM_TOTAL}*100" | bc | grep -oE "[0-9]*" | awk 'NR==1 {print; exit}'))
120 | if (( $(echo "${RAM_PERC} > ${RAM_ALARM}" | bc -l) )); then
121 | SEND=1
122 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"_${RAM_T::-1} ${RAM_PERC}%.\n"
123 | else
124 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"${RAM_T} ${RAM_PERC}%.\n"
125 | fi
126 |
127 | # get SWAP load
128 | SWAP_TOTAL=$(cat ~/temp.txt | grep "Swap" | awk '{print $2}')"G"
129 | SWAP_USED=$(cat ~/temp.txt | grep "Swap" | awk '{print $3}')"G"
130 | SWAP_PERC=$(printf "%.0f" $(echo "scale=2; ${SWAP_USED}/${SWAP_TOTAL}*100" | bc | grep -oE "[0-9]*" | awk 'NR==1 {print; exit}'))
131 | if [[ ${SWAP_TOTAL} != "0G" && ${SWAP_TOTAL} != "G" && ${SWAP_USED} != "0G" && ${SWAP_USED} != "G" ]]; then
132 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"${SWAP_T} ${SWAP_PERC}%.\n"
133 | fi
134 |
135 | # get disk load
136 | df -h / > ~/temp.txt
137 | DISK_PERC=$(printf "%.0f" $(cat ~/temp.txt | awk '{print $5}' | awk 'NR==2 {print; exit}' | tr -d '%'))
138 | if (( $(echo "${DISK_PERC} > ${PARTITION_ALARM}" | bc -l) )); then
139 | SEND=1
140 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"_${PART_T::-1} ${DISK_PERC}%.\n"
141 | else
142 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"${PART_T} ${DISK_PERC}%.\n"
143 | fi
144 |
145 | # get system load
146 | SYSTEM_LOAD=$(cat /proc/loadavg | awk '{print $2}')
147 | CPU_LOAD_MESSAGE=${CPU_LOAD_MESSAGE}"${LOAD_T} ${SYSTEM_LOAD}.\n"
148 |
149 | __DiskVitality
150 |
151 | # delete the temp file
152 | rm ~/temp.txt
153 |
154 | # if 'SEND' == 1 > send 'MESSAGE' into 'alarm telegram channel'
155 | if [[ ${SEND} == "1" ]]; then
156 | TEXT=${CPU_LOAD_MESSAGE}
157 | __Send
158 |
159 | curl --header 'Content-Type: application/json' \
160 | --request 'POST' \
161 | --data '{"chat_id":"'"${CHAT_ID_ALARM}"'", "text":"'"$(echo -e "${MESSAGE}")"'", "parse_mode": "html"}' "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
162 | > /dev/null 2>&1
163 | else
164 | echo -e "${CPU_LOAD_MESSAGE}"
165 | fi
166 | }
167 |
168 | function __LastChainBlock() {
169 | # get the last explorer's block
170 | if [[ ${CURL} == *"v1/status"* ]]; then
171 | LATEST_CHAIN_BLOCK=$(curl -sk ${CURL} | jq ".block_height" | tr -d '"')
172 | if [[ ${LATEST_CHAIN_BLOCK} == "null" ]]; then
173 | LATEST_CHAIN_BLOCK=$(curl -sk ${CURL} | jq ".data.block_height" | tr -d '"')
174 | fi
175 | elif [[ ${CURL} == *"bank/total"* ]] || [[ ${CURL} == *"blocks/latest"* ]]; then
176 | LATEST_CHAIN_BLOCK=$(curl -sk ${CURL} | jq ".height" | tr -d '"')
177 | if [[ ${LATEST_CHAIN_BLOCK} == "null" ]]; then
178 | LATEST_CHAIN_BLOCK=$(curl -sk ${CURL} | jq ".block.header.height" | tr -d '"')
179 | fi
180 | elif [[ ${CURL} == *"block?latest"* ]]; then
181 | LATEST_CHAIN_BLOCK=$(curl -sk ${CURL} | jq ".result.block.header.height" | tr -d '"')
182 | else
183 | LATEST_CHAIN_BLOCK="0"
184 | fi
185 | echo ${LATEST_CHAIN_BLOCK}
186 | }
187 |
188 | function __SignedAndMissedBlocks() {
189 | # init some variables
190 | SIGNED=0
191 | MISSED=0
192 | MAX_ROW=0
193 | LOOKBEHIND_BLOCKS=100
194 |
195 | # get slashing params
196 | SLASHING=$(${COSMOS} q slashing params -o json --node ${NODE} --home ${NODE_HOME})
197 | WINDOW=$(echo ${SLASHING} | jq ".signed_blocks_window" | tr -d '"')
198 | MIN_SIGNED=$(echo ${SLASHING} | jq ".min_signed_per_window" | tr -d '"')
199 | JAILED_AFTER=$(echo ${WINDOW}-${WINDOW}*${MIN_SIGNED} | bc -l | grep -oE "[0-9]*" | awk 'NR==1 {print; exit}')
200 | # LOOKBEHIND_BLOCKS=${JAILED_AFTER}
201 | MISSED_BLOCKS_FOR_ALARM=$(echo ${JAILED_AFTER}/10 | bc -l | grep -oE "[0-9]*" | awk 'NR==1 {print; exit}')
202 | # echo ${MISSED_BLOCKS_FOR_ALARM}
203 |
204 | # get some info about node
205 | NODE_STATUS_TOTAL=$(curl -s localhost:${PORT}/status)
206 | VALIDATOR_ADDRESS=$(echo ${NODE_STATUS_TOTAL} | jq .result.validator_info.address | tr -d '"')
207 | LATEST_BLOCK_HEIGHT=$(echo ${NODE_STATUS_TOTAL} | jq .result.sync_info.latest_block_height | tr -d '"')
208 | LATEST_BLOCK_TIME=$(echo ${NODE_STATUS_TOTAL} | jq .result.sync_info.latest_block_time | tr -d '"')
209 |
210 | # check only 100 last blocks
211 | START_BLOCK=$((${LATEST_BLOCK_HEIGHT}-${LOOKBEHIND_BLOCKS}+1))
212 | for (( BLOCK = ${START_BLOCK}; BLOCK <= ${LATEST_BLOCK_HEIGHT}; BLOCK++ )); do
213 | # check for validator signature
214 | SIGNATURE=$(curl -s localhost:${PORT}/block?height=${BLOCK} | jq .result.block.last_commit.signatures[].validator_address | tr -d '"' | grep ${VALIDATOR_ADDRESS})
215 |
216 | # if signature exists > signed + 1
217 | if [[ ${SIGNATURE} != "" ]]; then
218 | MISSED_IN_A_ROW=0
219 | ((SIGNED=SIGNED+1))
220 |
221 | # if signature does not exist > missed + 1
222 | else
223 | ((MISSED_IN_A_ROW=MISSED_IN_A_ROW+1))
224 | ((MISSED=MISSED+1))
225 | if (( ${MISSED_IN_A_ROW} > ${MAX_ROW} )); then MAX_ROW=${MISSED_IN_A_ROW}; fi
226 | fi
227 | done
228 |
229 | # if missed is more than 10 > check the whole window before jail
230 | if [[ ${MISSED} > "10" ]]; then
231 |
232 | # zeroing variables
233 | SIGNED=0
234 | MISSED=0
235 | MAX_ROW=0
236 | LOOKBEHIND_BLOCKS=${JAILED_AFTER}
237 |
238 | START_BLOCK=$((${LATEST_BLOCK_HEIGHT}-${LOOKBEHIND_BLOCKS}+1))
239 | for (( BLOCK = ${START_BLOCK}; BLOCK <= ${LATEST_BLOCK_HEIGHT}; BLOCK++ )); do
240 | # check for validator signature
241 | SIGNATURE=$(curl -s localhost:${PORT}/block?height=${BLOCK} | jq .result.block.last_commit.signatures[].validator_address | tr -d '"' | grep ${VALIDATOR_ADDRESS})
242 |
243 | # if signature exists > signed + 1
244 | if [[ ${SIGNATURE} != "" ]]; then
245 | MISSED_IN_A_ROW=0
246 | ((SIGNED=SIGNED+1))
247 |
248 | # if signature does not exist > missed + 1
249 | else
250 | ((MISSED_IN_A_ROW=MISSED_IN_A_ROW+1))
251 | ((MISSED=MISSED+1))
252 | if (( ${MISSED_IN_A_ROW} > ${MAX_ROW} )); then MAX_ROW=${MISSED_IN_A_ROW}; fi
253 | fi
254 | done
255 | fi
256 |
257 | # if missed more than 10% of allowed missed blocks before jail > alarm
258 | if (( ${MISSED} > ${MISSED_BLOCKS_FOR_ALARM} )); then
259 | SEND=1
260 | TEXT="_${MISSED_T::-1} ${MISSED}/${LOOKBEHIND_BLOCKS} last blocks.\n_${TRAIN_T::-1} ${MAX_ROW} missed in a row.\n${JAILED_A} after ${JAILED_AFTER} missed blocks."
261 | __Send
262 | elif (( ${MISSED} > 0 )); then
263 | TEXT="${MISSED_T} ${MISSED} blocks.\n${TRAIN_T} ${MAX_ROW} missed in a row."
264 | __Send
265 | else
266 | echo "${MISSED_T} ${MISSED} blocks."
267 | fi
268 | }
269 |
270 | function __UnvotedProposals() {
271 |
272 | # get proposals
273 | PROPOSALS=$(${COSMOS} q gov proposals --node ${NODE} --limit 999999999 --output json 2>&1)
274 |
275 | # if at least one proposal exists
276 | if [[ ${PROPOSALS} != *"no proposals found"* ]]; then
277 | # get array of active proposals
278 | ACTIVE_PROPOSALS_STRING=$(echo ${PROPOSALS} | jq '.proposals[] | select(.status=="PROPOSAL_STATUS_VOTING_PERIOD")' | jq '.proposal_id' | tr -d '"')
279 | ACTIVE_PROPOSALS_ARRAY=($(echo "${ACTIVE_PROPOSALS_STRING}" | tr ' ' '\n'))
280 |
281 | # init array of unvoted proposals
282 | UNVOTED_ARRAY=( )
283 |
284 | # run loop on each proposal
285 | for i in "${!ACTIVE_PROPOSALS_ARRAY[@]}"; do
286 | # if vote does not exist, add proposal id to 'UNVOTED_ARRAY'
287 | VOTE=$(${COSMOS} q gov votes ${ACTIVE_PROPOSALS_ARRAY[i]} --limit 999999999 --node ${NODE} --output json | jq '.votes[].voter' | tr -d '"' | grep ${DELEGATOR_ADDRESS})
288 | if [[ ${VOTE} == "" ]]; then UNVOTED_ARRAY+=(${ACTIVE_PROPOSALS_ARRAY[i]}); fi
289 | done
290 |
291 | # if exists at least one unvoted proposal
292 | if (( ${#UNVOTED_ARRAY[@]} > 0 )); then
293 | TEXT="_${GOV_T::-1}"
294 |
295 | # add proposal id to message
296 | for i in "${!UNVOTED_ARRAY[@]}"; do
297 | TEXT=${TEXT}' #'${UNVOTED_ARRAY[i]}
298 | if (( ${i} < ${#UNVOTED_ARRAY[@]}-1 )); then TEXT=${TEXT}','; else TEXT=${TEXT}'.'; fi
299 | done
300 | __Send
301 | else
302 | echo "${GOV_T} no unvoted proposals."
303 | fi
304 | else
305 | echo "${GOV_T} no any proposals."
306 | fi
307 | }
308 |
309 | function __AverageBlockExecutionTime() {
310 |
311 | # init some variables
312 | LOOKBEHIND_BLOCKS=100
313 |
314 | # get some info about node
315 | NODE_STATUS_TOTAL=$(curl -s localhost:${PORT}/status)
316 |
317 | # get last block time and height
318 | LATEST_BLOCK_HEIGHT=$(echo ${NODE_STATUS_TOTAL} | jq .result.sync_info.latest_block_height | tr -d '"')
319 | LATEST_BLOCK_TIME=$(echo ${NODE_STATUS_TOTAL} | jq .result.sync_info.latest_block_time | tr -d '"' | grep -oE "[0-9]*:[0-9]*:[0-9]*")
320 | IFS=':' read -ra HMS <<< "$LATEST_BLOCK_TIME"
321 | LATEST_BLOCK_TIME_IN_SEC=$(echo ${HMS[0]}*3600+${HMS[1]}*60+${HMS[2]} | bc -l)
322 |
323 | # get upgrade block time and height
324 | UPGRADE_BLOCK_HEIGHT=$((${LATEST_BLOCK_HEIGHT}-${LOOKBEHIND_BLOCKS}))
325 | UPGRADE_BLOCK_TIME=$(${COSMOS} q block ${UPGRADE_BLOCK_HEIGHT} --node ${NODE} | jq ".block.header.time" | tr -d '"' | grep -oE "[0-9]*:[0-9]*:[0-9]*")
326 | IFS=':' read -ra HMS <<< "${UPGRADE_BLOCK_TIME}"
327 | UPGRADE_BLOCK_TIME_IN_SEC=$(echo ${HMS[0]}*3600+${HMS[1]}*60+${HMS[2]} | bc -l)
328 |
329 | # find the max and the min values
330 | MAX=${LATEST_BLOCK_TIME_IN_SEC}; MIN=${LATEST_BLOCK_TIME_IN_SEC}
331 | if (( ${UPGRADE_BLOCK_TIME_IN_SEC} > ${MAX} )); then MAX=${UPGRADE_BLOCK_TIME_IN_SEC}; fi
332 | if (( ${UPGRADE_BLOCK_TIME_IN_SEC} < ${MIN} )); then MIN=${UPGRADE_BLOCK_TIME_IN_SEC}; fi
333 |
334 | # find the difference between blocks in seconds
335 | DIFF_IN_SEC=$((${MAX}-${MIN}))
336 | if (( $(echo "86400 - ${DIFF_IN_SEC}" | bc) < ${DIFF_IN_SEC} )); then
337 | DIFF_IN_SEC=$((86400 - ${DIFF_IN_SEC})); echo ${DIFF_IN_SEC}
338 | fi
339 |
340 | # get estimated block exectuion time
341 | BLOCK_EXECUTION_TIME=$(echo "scale=2;${DIFF_IN_SEC}/100" | bc)
342 | }
343 |
344 | function __UpgradePlan() {
345 |
346 | # init some variables
347 | UPGRADE_ALARM_IN_MIN=30
348 | ETBU_TEXT=""
349 | ETBU_APPR=""
350 |
351 | # read 'cosmos.conf'
352 | . ./cosmos.conf
353 |
354 | # get some info about chain upgrade plan
355 | UPGRADE_PLAN=$(${COSMOS} q upgrade plan --node ${NODE} --output json 2>&1)
356 |
357 | # if smth is planned, then calculate approximate upgrade time
358 | if [[ ${UPGRADE_PLAN} != *"no upgrade scheduled"* ]]; then
359 | UPGRADE_HEIGHT=$(echo ${UPGRADE_PLAN} | jq ".height" | tr -d '"')
360 | UPGRADE_NAME=$(echo ${UPGRADE_PLAN} | jq ".name" | tr -d '"')
361 |
362 | __AverageBlockExecutionTime
363 | BLOCKS_BEFORE_UPGRADE=$((${UPGRADE_HEIGHT}-${LATEST_BLOCK_HEIGHT}))
364 | ESTIMATED_TIME_BEFORE_UPGRADE_IN_SEC=$(printf "%.0f" $(echo "scale=2; ${BLOCKS_BEFORE_UPGRADE}*${BLOCK_EXECUTION_TIME}" | bc))
365 | ESTIMATED_TIME_BEFORE_UPGRADE_IN_MIN=$(echo "scale=2; ${BLOCKS_BEFORE_UPGRADE}*${BLOCK_EXECUTION_TIME}/60" | bc)
366 |
367 | ETBU_D=$((${ESTIMATED_TIME_BEFORE_UPGRADE_IN_SEC}/60/60/24))
368 | ETBU_H=$((${ESTIMATED_TIME_BEFORE_UPGRADE_IN_SEC}/60/60%24))
369 | ETBU_M=$((${ESTIMATED_TIME_BEFORE_UPGRADE_IN_SEC}/60%60))
370 |
371 | if (( ${ETBU_D} > 0 )); then ETBU_TEXT="${ETBU_TEXT}${ETBU_D}d "; fi
372 | if (( ${ETBU_D} > 0 )); then ETBU_APPR="${ETBU_APPR}${ETBU_D}day "; fi
373 | if (( ${ETBU_H} > 0 )); then ETBU_TEXT="${ETBU_TEXT}${ETBU_H}h "; fi
374 | if (( ${ETBU_H} > 0 )); then ETBU_APPR="${ETBU_APPR}${ETBU_H}hour "; fi
375 | if (( ${ETBU_M} > 0 )); then ETBU_TEXT="${ETBU_TEXT}${ETBU_M}m "; fi
376 | if (( ${ETBU_M} > 0 )); then ETBU_APPR="${ETBU_APPR}${ETBU_M}min "; fi
377 |
378 | APPROXIMATE_UPGRADE_TIME=$(date -d "+${ETBU_APPR}" +"%b %d, %H:%M")
379 |
380 | if [[ $(echo "${ESTIMATED_TIME_BEFORE_UPGRADE_IN_MIN} < ${UPGRADE_ALARM_IN_MIN}" | bc) -eq 1 && ${ESTIMATED_TIME_BEFORE_UPGRADE_IN_MIN} != 0 ]]; then
381 | SEND=1
382 | TEXT="\n_${UPGRADE_T::-1} ${UPGRADE_NAME}.\n${TIME_L_T} ${ETBU_TEXT::-1}.\n${APPR_T_T} ${APPROXIMATE_UPGRADE_TIME}."
383 | __Send
384 | else
385 | TEXT="\n_${UPGRADE_T::-1} ${UPGRADE_NAME}.\n${TIME_L_T} ${ETBU_TEXT::-1}.\n${APPR_T_T} ${APPROXIMATE_UPGRADE_TIME}."
386 | __Send
387 | fi
388 | else
389 | echo "${UPGRADE_T} no."
390 | fi
391 | }
392 |
393 | function __NodeStatus() {
394 |
395 | # init some variables
396 | MESSAGE="${PROJECT} ⠀|⠀ ${MONIKER}\n\n"
397 | echo -e "${PROJECT} | ${MONIKER}\n"
398 | INACTIVE=""
399 | SEND=0
400 |
401 | # get some info about node
402 | NODE_STATUS=$(timeout 5s ${COSMOS} status 2>&1 --node ${NODE} --home ${NODE_HOME})
403 |
404 | # if 'NODE_STATUS' response contains 'connection refused' > instant alarm
405 | if [[ ${NODE_STATUS} != *"connection refused"* ]] && [[ ${NODE_STATUS} != "" ]]; then
406 | # get the lastest node and explorer blocks height
407 | LATEST_NODE_BLOCK=$(echo ${NODE_STATUS} | jq .'SyncInfo'.'latest_block_height' | tr -d '"')
408 | LATEST_CHAIN_BLOCK=$(__LastChainBlock)
409 |
410 | # if 'CURL' was not set > no compare with explorer height
411 | if [[ ${CURL} != "" ]] && [[ ${LATEST_CHAIN_BLOCK} != "" ]] && [[ ${LATEST_CHAIN_BLOCK} != "0" ]] && [[ ${LATEST_CHAIN_BLOCK} != "null" ]]; then
412 | # if we are in the past more than 100 block > alarm
413 | if ((${LATEST_CHAIN_BLOCK}-${BLOCK_GAP_ALARM} > ${LATEST_NODE_BLOCK})); then
414 | if [[ ${ALLOW_SERVICE_RESTART} == "true" ]]; then
415 | systemctl restart ${SERVICE} > /dev/null 2>&1
416 | TEXT="_${EXP_ME_T::-1} ${LATEST_CHAIN_BLOCK}/${LATEST_NODE_BLOCK}. \n\nbut service has been restarted."
417 | else
418 | TEXT="_${EXP_ME_T::-1} ${LATEST_CHAIN_BLOCK}/${LATEST_NODE_BLOCK}."
419 | fi
420 | SEND=1
421 | __Send
422 | else
423 | echo "${EXP_ME_T} ${LATEST_CHAIN_BLOCK}/${LATEST_NODE_BLOCK}."
424 | fi
425 | else
426 | echo "${EXP_ME_T} 0/${LATEST_NODE_BLOCK}."
427 | fi
428 |
429 | # if there is no problem with height
430 | if [[ ${SEND} == "0" ]]; then
431 |
432 | # check 'priv_key' actuality
433 | CONSENSUS_PUBKEY=$(${COSMOS} q staking validator ${VALIDATOR_ADDRESS} -oj --node ${NODE} --home ${NODE_HOME} | jq -r ".consensus_pubkey.key")
434 | CURRENT_PUBKEY=$(curl -s localhost:${PORT}/status | jq -r ".result.validator_info.pub_key.value")
435 | if [[ ${CONSENSUS_PUBKEY} == ${CURRENT_PUBKEY} ]]; then
436 | PRIVKEY="right"
437 | echo "${PRIVKEY_T} right."
438 | else
439 | PRIVKEY="wrong"
440 | if [[ ${IGNORE_WRONG_PRIVKEY} == "true" ]]; then
441 | TEXT="_${PRIVKEY_T::-1} wrong, but we know."
442 | __Send
443 | else
444 | SEND=1
445 | TEXT="_${PRIVKEY_T::-1} wrong."
446 | __Send
447 | fi
448 | fi
449 |
450 | # get validator info
451 | VALIDATOR_INFO=$(${COSMOS} query staking validator ${VALIDATOR_ADDRESS} --node ${NODE} --output json --home ${NODE_HOME})
452 | BOND_STATUS=$(echo ${VALIDATOR_INFO} | jq .'status' | tr -d '"')
453 |
454 | # if 'BOND_STATUS' is different than 'BOND_STATUS_BONDED' > alarm
455 | if [[ "${BOND_STATUS}" != "BOND_STATUS_BONDED" ]]; then
456 | # if 'JAILED_STATUS' is 'true' > alarm with 'jailed > true.'
457 | # if 'JAILED_STATUS' is 'true' > alarm with 'active > false.'
458 | INACTIVE="true"
459 |
460 | JAILED_STATUS=$(echo ${VALIDATOR_INFO} | jq .'jailed')
461 | if [[ "${JAILED_STATUS}" == "true" ]]; then
462 | if [[ ${IGNORE_WRONG_PRIVKEY} == "true" && ${PRIVKEY} == "wrong" ]]; then
463 | TEXT="${JAILED_A} ${JAILED_STATUS}, but we know."
464 | else
465 | SEND=1
466 | TEXT="${JAILED_A} ${JAILED_STATUS}."
467 | fi
468 | else
469 | # if 'ignore_inactive_status' is not set or 'false' > alarm
470 | if [[ ${IGNORE_INACTIVE_STATUS} != "true" ]]; then SEND=1; fi
471 | TEXT="${ACTIVE_A} false."
472 | fi
473 | __Send
474 |
475 | # if 'BOND_STATUS' is 'BOND_STATUS_BONDED' > continue
476 | else
477 | # get local explorer snapshot and request some info about our validator
478 | EXPLORER=$(${COSMOS} q staking validators --node ${NODE} --output json --home ${NODE_HOME} --limit=999999999)
479 | VALIDATORS_COUNT=$(echo ${EXPLORER} | jq '.validators[] | select(.status=="BOND_STATUS_BONDED")' | jq -r '.tokens' | sort -gr | wc -l)
480 | VALIDATOR_STRING=$(echo ${EXPLORER} | jq '.validators[] | select(.status=="BOND_STATUS_BONDED")' | jq -r '.tokens + " " + .description.moniker' | sort -gr | nl | grep -F ${MONIKER})
481 | VALIDATOR_POSITION=$(echo ${VALIDATOR_STRING} | awk '{print $1}')
482 | ACTIVE_VALIDATOR_SET=$(${COSMOS} q staking params --node ${NODE} --output json --home ${NODE_HOME} | jq ."max_validators")
483 |
484 | # alarm if validator is close to become inactive
485 | SAFE_VALIDATOR_PLACE=$(echo ${ACTIVE_VALIDATOR_SET}-${POSITION_GAP_ALARM} | bc -l)
486 |
487 | if ((${VALIDATOR_POSITION} > ${SAFE_VALIDATOR_PLACE})); then
488 | if [[ ${IGNORE_INACTIVE_STATUS} != "true" ]]; then
489 | SEND=1
490 | TEXT="_${PLACE_T::-1} ${VALIDATOR_POSITION}/${ACTIVE_VALIDATOR_SET}."
491 | else
492 | TEXT="${PLACE_T} ${VALIDATOR_POSITION}/${ACTIVE_VALIDATOR_SET}."
493 | fi
494 | else
495 | TEXT="${PLACE_T} ${VALIDATOR_POSITION}/${ACTIVE_VALIDATOR_SET}."
496 | fi
497 | __Send
498 |
499 | # validator active stake
500 | VALIDATOR_STAKE=$(echo ${VALIDATOR_STRING} | awk '{print $2}')
501 | TEXT="${STAKE_T} $(echo "scale=2;${VALIDATOR_STAKE}/${DENOM}" | bc) ${TOKEN}."
502 | __Send
503 | fi
504 |
505 | if [[ ${INACTIVE} != "true" ]]; then
506 | if [[ ${IGNORE_WRONG_PRIVKEY} != "true" ]]; then
507 | __SignedAndMissedBlocks
508 | else
509 | echo "missed >>>>> ignore, cause of 'IWP'."
510 | fi
511 | else
512 | echo "missed >>>>> ignore, cause of inactive status."
513 | fi
514 |
515 | # get info about proposals
516 | __UnvotedProposals
517 |
518 | # get info about upgrades
519 | __UpgradePlan
520 | fi
521 | else
522 | if [[ ${NODE_STATUS} == "" ]]; then
523 | if [[ ${ALLOW_SERVICE_RESTART} == "true" ]]; then
524 | systemctl restart ${SERVICE} > /dev/null 2>&1
525 | TEXT="_we lost any connection. \n\nbut service has been restarted."
526 | else
527 | TEXT="_we lost any connection."
528 | fi
529 | else
530 | if [[ ${ALLOW_SERVICE_RESTART} == "true" ]]; then
531 | systemctl restart ${SERVICE} > /dev/null 2>&1
532 | TEXT="_connection is refused. \n\nbut service has been restarted."
533 | else
534 | TEXT="_connection is refused."
535 | fi
536 | fi
537 |
538 | SEND=1
539 | __Send
540 | fi
541 |
542 | # read the config
543 | # echo ${CONF}
544 | . ./${CONF}
545 |
546 | # if 'SEND' == 1 > send 'MESSAGE' into 'alarm telegram channel'
547 | if [[ ${SEND} == "1" ]]; then
548 | curl --header 'Content-Type: application/json' \
549 | --request 'POST' \
550 | --data '{"chat_id":"'"${CHAT_ID_ALARM}"'", "text":"'"$(echo -e "${MESSAGE}")"'", "parse_mode": "html"}' "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
551 | > /dev/null 2>&1
552 | fi
553 |
554 | # send 'log_message' only one time per hour
555 | # if the current minute value is less than 'MINUTE' > send 'log_message', else > ignore
556 | if (( $(echo "$(date +%M) < ${MINUTE}" | bc -l) )); then
557 | # write message text into temp file to send all logs via one message
558 | echo ${MESSAGE} >> "./${CHAT_ID_STATUS}"
559 | fi
560 | }
561 |
562 | function Main() {
563 | # init some variables
564 | SEND_LOAD=""
565 | MINUTE=10
566 | TIMEZONE="Africa/Abidjan"
567 | DISK_PERCENTAGE_USED_ALARM=100
568 | __PreMessage
569 |
570 | # print the current time
571 | echo -e " "; echo -e "/// $(date '+%F %T') ///"; echo -e " "
572 |
573 | # read 'cosmos.conf'
574 | cd $HOME/status/ && . ./cosmos.conf
575 | export TZ=${TIMEZONE}
576 |
577 | # get ServerLoad info
578 | __ServerLoad
579 |
580 | # run 'NodeStatus' with every '*.conf' file in the 'status' folder
581 | for CONF in *.conf; do
582 |
583 | # if config at least contains 'COSMOS' string, then go
584 | if [[ $(cat ${CONF}) == *"COSMOS"* ]]; then
585 |
586 | # init some variables
587 | IGNORE_INACTIVE_STATUS=""
588 | IGNORE_WRONG_PRIVKEY=""
589 | ALLOW_SERVICE_RESTART=""
590 | POSITION_GAP_ALARM=0
591 | BLOCK_GAP_ALARM=100
592 |
593 | # read the config
594 | . ./${CONF}
595 | echo -e " "
596 |
597 | # if config directory, config.toml and genesis.json exist
598 | if [[ -e "${CONFIG}" && -e "${CONFIG}/config.toml" && -e "${CONFIG}/genesis.json" ]]; then
599 |
600 | # get '--node' and '--chain' value
601 | NODE=$(cat ${CONFIG}/config.toml | grep -oPm1 "(?<=^laddr = \")([^%]+)(?=\")")
602 | NODE_HOME=$(echo ${CONFIG} | rev | cut -c 8- | rev)
603 | CHAIN=$(cat ${CONFIG}/genesis.json | jq .chain_id | sed -E 's/.*"([^"]+)".*/\1/')
604 | PORT=$(echo ${NODE} | awk 'NR==1 {print; exit}' | grep -o ":[0-9]*" | awk 'NR==2 {print; exit}' | cut -c 2-)
605 |
606 | # run 'NodeStatus'
607 | __NodeStatus
608 | else
609 | echo -e "${PROJECT} | ${MONIKER}\n"
610 | echo "we have some problems with config. maybe config files do not exist."
611 | MESSAGE="${PROJECT} ⠀|⠀ ${MONIKER}\n\nwe have some problems with config.\nremove '${CONF}' or fix it.\n\n"
612 |
613 | # send 'alarm_message'
614 | curl --header 'Content-Type: application/json' \
615 | --request 'POST' \
616 | --data '{"chat_id":"'"${CHAT_ID_ALARM}"'", "text":"'"$(echo -e "${MESSAGE}")"'", "parse_mode": "html"}' "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
617 | > /dev/null 2>&1
618 | fi
619 | fi
620 | done
621 |
622 | # send 'log_messages' from temp files into chats
623 | CHAT_IDS=($(ls | grep -E "[0-9]{2,}"))
624 | for i in "${!CHAT_IDS[@]}"; do
625 | SLASH="/// $(date '+%F %T') ///"
626 |
627 | curl --header 'Content-Type: application/json' \
628 | --request 'POST' \
629 | --data '{"chat_id":"'"${CHAT_IDS[i]}"'","text":"'"$(echo -e ${SLASH})"'", "parse_mode": "html"}' "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
630 | > /dev/null 2>&1
631 |
632 | curl --header 'Content-Type: application/json' \
633 | --request 'POST' \
634 | --data '{"chat_id":"'"${CHAT_IDS[i]}"'", "text":"'"$(cat "./${CHAT_IDS[i]}")"'", "parse_mode": "html"}' "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
635 | > /dev/null 2>&1
636 | done
637 |
638 | # delete temp log files
639 | rm ./-?[0-9]* > /dev/null 2>&1
640 | }
641 |
642 | # run 'main'
643 | Main
644 |
--------------------------------------------------------------------------------
/tendermint/node_status/crontab:
--------------------------------------------------------------------------------
1 | # status
2 | 1,11,21,31,41,51 * * * * bash $HOME/status/cosmos.sh >> $HOME/status/cosmos.log 2>&1
3 |
--------------------------------------------------------------------------------
/tendermint/node_status/curl.md:
--------------------------------------------------------------------------------
1 | **PROJECT** | **CURL** |
2 | --- | --- |
3 | gravity-bridge-m | CURL="https://api-gravity-bridge.cosmostation.io/v1/status" |
4 | sifchain-m | CURL="https://api-sifchain.cosmostation.io/v1/status" |
5 | omniflix-m | CURL="https://api-omniflix.cosmostation.io/v1/status" |
6 | provenance-m | CURL="https://api-provenance.cosmostation.io/v1/status" |
7 | evmos-m | CURL="https://api-evmos.cosmostation.io/v1/status" |
8 | galaxy-m | CURL="https://galaxy.postcapitalist.io/bank/total/uglx" |
9 | meme-m | CURL="https://api-meme-1.meme.sx/bank/total/umeme" |
10 | beezee-m | CURL="https://rest.getbze.com/bank/total/ubze" |
11 | desmos-m | CURL="https://api-desmos.cosmostation.io/v1/status" |
12 | vidulum-m | CURL="https://mainnet-lcd.vidulum.app/bank/total/uvdl" |
13 | chromic-m | CURL="https://chtd-api.skynetvalidators.com/bank/total/ucht" |
14 | dig-m | CURL="https://api-1-dig.notional.ventures/bank/total/udig" |
15 | konstellation-m | CURL="https://api-konstellation.cosmostation.io/v1/status" |
16 | mises-m | CURL="https://rpc.gw.mises.site/block?latest" |
17 | idep-m | CURL="https://rpc.idep.sgtstake.com/block?latest" |
18 | bitsong-m | CURL="https://rpc.bitsong.forbole.com/block?latest" |
19 | | |
20 | kyve-t | CURL="https://api.explorer.kyve.network/bank/total/tkyve" |
21 | stafi-t | CURL="https://test-rest-rpc1.stafihub.io/bank/total/ufis" |
22 | pylons-t | CURL="https://pylons.api.explorers.guru/api/blocks/latest" |
23 | dws-t | CURL="https://dws.api.explorers.guru/api/blocks/latest" |
24 | quicksilver-t | CURL="https://quicksilver.api.explorers.guru/api/blocks/latest" |
25 | kqcosmos-t | CURL="https://api-3.quick.testnet.run/bank/total/uatom" |
26 | defund-t | CURL="https://defund.api.explorers.guru/api/blocks/latest" |
27 | sei-t | CURL="https://sei.api.explorers.guru/api/blocks/latest" |
28 | celestia-t | CURL="https://celestia.api.explorers.guru/api/blocks/latest" |
29 | another-t | CURL="https://api-anone.notional.ventures/bank/total/uan1" |
30 | uptik-t | CURL="https://seed1.testnet.uptick.network:1318/bank/total" |
31 | cosmic-t | CURL="https://coho.api.explorers.guru/api/blocks/latest" |
32 | aura-t | CURL="https://halo-api.aurascan.io/api/v1/status" |
33 | clan-t | CURL="https://api.clan.testnet.run/bank/total/uclan" |
34 | kujira-t | CURL="https://kujira-testnet.synergynodes.com/bank/total/ukuji" |
35 |
36 | If you want to add your custom CURL into script, you have to open the needed explorer, open `Inspect` with your mouse's right click, open `Network` tab and find a request with `v1/status`, `bank/total` or `blocks/latest` words in the request link.
37 |
38 | Or just ping `@cyberomanov` via telegram.
39 |
--------------------------------------------------------------------------------
/tendermint/node_status/name.conf:
--------------------------------------------------------------------------------
1 | # /////////////////////////////////////////////////////////////////////////////////////////////////
2 | # ////////////////// important variables for script logic, required to be filled //////////////////
3 | # /////////////////////////////////////////////////////////////////////////////////////////////////
4 |
5 | # validator moniker
6 | MONIKER="cyberomanov"
7 | # delegator address
8 | DELEGATOR_ADDRESS="cosmos12af...24"
9 | # validator address
10 | VALIDATOR_ADDRESS="cosmosvaloper12af...24"
11 | # token name
12 | TOKEN="atom"
13 | # token denomination (count of nulls)
14 | DENOM=1000000
15 | # project name, used in 'log_messages' and 'alarm_messages' for easy reading
16 | PROJECT="cosmos-t"
17 | # exact full path to bin
18 | COSMOS="/root/go/bin/cosmosd"
19 | # exact full path to config folder
20 | CONFIG="/root/.cosmos/config/"
21 | # 'chat_id' for 'alarm_messages' with enabled notifications
22 | CHAT_ID_ALARM="-11223344"
23 | # 'chat_id' for 'log_messages'
24 | CHAT_ID_STATUS="-44332211"
25 | # 'bot_token' for sending messages
26 | BOT_TOKEN="55667788:xxx-xxx_xxx"
27 |
28 | # /////////////////////////////////////////////////////////////////////////////////////////////////
29 | # /////// custom configuration, uncomment a specific variable to enable a specific function ///////
30 | # /////////////////////////////////////////////////////////////////////////////////////////////////
31 |
32 | # 1. link to an explorer API to get a difference between 'ideal_latest_block' and 'local_latest_block'
33 | #
34 | # if validator is in the past more than 'N' blocks > 'alarm_message'
35 | # try to find your 'curl' in 'curl.md' file or ping @cyberomanov via telegram for help
36 | #
37 | # uncomment the following variable and set your value to enable the function, disabled by default
38 | # CURL="https://api-cosmos.cosmostation.io/v1/status"
39 |
40 | # 2. definition of 'N'
41 | #
42 | # doesn't work without correctly filled 'CURL' value
43 | #
44 | # examples:
45 | # conditions: 'CURL' is set, 'BLOCK_GAP_ALARM' is set to '100' and 'ideal_latest_block' is 21000
46 | # result #1: if 'local_latest_block' is 21003 > no 'alarm_message'
47 | # result #2: if 'local_latest_block' is 20997 > no 'alarm_message'
48 | # result #3: if 'local_latest_block' is 20895 > 'alarm_message'
49 | #
50 | # uncomment the following variable and set your value to enable the function, set '0' to disable
51 | # BLOCK_GAP_ALARM=100
52 |
53 | # -------------------------------------------------------------------------------------------------
54 |
55 | # 3. acceptable gap between validator position and max set of active validators
56 | #
57 | # examples:
58 | # conditions: max set is 200 active validators and 'POSITION_GAP_ALARM' is set to '10'
59 | # result #1: if validator place is from 1st to 190th > no 'alarm_message'
60 | # result #2: if validator place is from 191st to 200th > 'alarm_message'
61 | #
62 | # uncomment the following variable and set your value to enable the function, disabled by default
63 | # POSITION_GAP_ALARM=10
64 |
65 | # -------------------------------------------------------------------------------------------------
66 |
67 | # 4. ignore alarm trigger when validator has inactive status
68 | #
69 | # uncomment the following variable, if you want to do ignore inactive status alarm trigger
70 | # or leave it commented, if you want to receive 'alarm_messages' about inactive status
71 | # IGNORE_INACTIVE_STATUS="true"
72 |
73 | # 5. ignore alarm trigger when validator has wrong 'priv_validator_key'
74 | #
75 | # if you know that validator is running with a wrong priv_key
76 | # than you may want to ignore 'jailed_status' and 'many_missed_blocks' trigger for 'alarm_messages'
77 | #
78 | # uncomment the following variable, if you want to do ignore mentioned alarm triggers
79 | # or leave it commented, if you want to receive 'alarm_messages' about jails/missed_blocks
80 | # IGNORE_WRONG_PRIVKEY="true"
81 |
82 | # -------------------------------------------------------------------------------------------------
83 |
84 | # 6. allow the script or not to allow to restart a specific service
85 | #
86 | # doesn't work without correctly filled 'SERVICE' value
87 | #
88 | # examples:
89 | # conditions #1: 'BLOCK_GAP_ALARM' is '100', 'ideal_latest_block' is 21000, 'ALLOW_SERVICE_RESTART' is 'true'
90 | # result #1: if 'local_latest_block' is 20895 > 'alarm_message' AND 'service_restart'
91 | # conditions #2: service is down and 'ALLOW_SERVICE_RESTART' is 'true'
92 | # result #2: 'alarm_message' AND 'service_restart'
93 | # conditions #3: service is up, but smth is wrong and 'ALLOW_SERVICE_RESTART' is 'true'
94 | # result #3: 'alarm_message' AND 'service_restart'
95 | #
96 | # uncomment the following variable, if you want to do 'service_restart' made by the script
97 | # or leave it commented, if you do not want to do 'service_restart' made by the script
98 | # ALLOW_SERVICE_RESTART="true"
99 |
100 | # 7. service name
101 | #
102 | # is not used anywhere if 'ALLOW_SERVICE_RESTART' is 'false' or commented
103 | # but used for 'service_restart' if 'ALLOW_SERVICE_RESTART' is 'true' or uncommented
104 | #
105 | # uncomment the following variable and set your value to enable the function
106 | # SERVICE="cosmosd"
107 |
--------------------------------------------------------------------------------