├── .gitignore ├── CNAME ├── LICENSE ├── README.md ├── _config.yml ├── ansible.cfg ├── docs ├── advanced.md ├── history.md ├── launch.gif ├── pitch.md ├── roadmap.md └── wizard.gif ├── install ├── debug.sh ├── install_agave_validator.sh ├── install_monitoring.sh ├── install_rpc_node.sh ├── install_update_sys_tuner.sh ├── install_validator.sh ├── restart_cluster.sh ├── update_agave_validator_version.sh ├── update_monitoring.sh ├── update_test_validator_version.sh └── update_validator_node.sh ├── inventory_example ├── group_vars │ ├── all.yml │ ├── local.yaml │ ├── mainnet_validators.yaml │ ├── remote.yaml │ ├── rpc.yaml │ └── testnet_validators.yaml ├── mainnet.yaml └── testnet.yaml ├── playbooks ├── pb_cluster_restart.yaml ├── pb_config.yaml ├── pb_debug.yaml ├── pb_install_agave_validator.yaml ├── pb_install_jito_validator.yaml ├── pb_install_monitoring.yaml ├── pb_install_validator.yaml ├── pb_stop_validator.yaml └── pb_update_validator.yaml └── roles ├── agave_cli └── tasks │ ├── install.yaml │ ├── main.yaml │ └── update.yaml ├── check_node └── tasks │ └── main.yaml ├── configure_ubuntu ├── tasks │ ├── ansible_user.yaml │ ├── cpu_governor.yaml │ ├── fail2ban.yaml │ ├── firewall.yaml │ ├── lvm.yaml │ ├── main.yaml │ ├── packages.yaml │ ├── ramdisk.yaml │ ├── solana_user.yaml │ └── swap.yaml └── templates │ └── jail.local.j2 ├── monitoring ├── files │ ├── cluster_monitoring_library.py │ ├── common.py │ ├── measurement_tds_info.py │ ├── measurement_validator_info.py │ ├── output_gossip.py │ ├── output_tds_measurements.py │ ├── output_validator_measurements.py │ ├── output_validators.py │ ├── output_validators_info.py │ ├── request_utils.py │ ├── solana_rpc.py │ ├── tds_info.py │ └── validator_monitoring.py ├── tasks │ ├── configure_telegraf.yaml │ ├── install_monitoring_script.yaml │ ├── install_telegraf.yaml │ ├── install_telegraf_rpc.yaml │ ├── main.yaml │ └── packages.yaml └── templates │ ├── monitoring_config.py.j2 │ ├── output_starter.sh.j2 │ ├── telegraf.conf.j2 │ └── telegraf.rpc.conf.j2 ├── restart_cluster ├── tasks │ └── main.yaml └── templates │ ├── create-snapshot.sh.j2 │ └── solana-validator.testnet.restart.service.j2 ├── solana_cli └── tasks │ ├── install.yaml │ ├── main.yaml │ └── update.yaml ├── solana_config ├── tasks │ └── main.yaml └── templates │ └── sv_manager.conf.template ├── solana_validator_bootstrap ├── tasks │ ├── cluster_environment.yaml │ ├── configure_validator.yaml │ ├── create_missing_keys.yaml │ ├── logrotate.yaml │ ├── main.yaml │ ├── solana-validator.service.yaml │ ├── sys-tuner.service.yaml │ ├── upload_keys.yaml │ └── validator_info.yaml └── templates │ ├── service.demo │ ├── solana-sys-tuner.service.j2 │ ├── solana-validator.logrotate.j2 │ └── solana-validator.service.j2 └── solana_validator_restart └── tasks ├── main.yaml └── wait_for_restart_window.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | .secrets/ 2 | .idea/ 3 | __pycache__/ 4 | inventory_backup/ 5 | inventory_test/ 6 | inventory_/ 7 | tmp/ 8 | inventory/mainnet.yaml 9 | inventory/testnet.yaml 10 | inventory/group_vars/all.yml 11 | inventory/group_vars/local.yaml 12 | inventory/group_vars/mainnet_validators.yaml 13 | inventory/group_vars/remote.yaml 14 | inventory/group_vars/testnet_validators.yaml 15 | inventory-test/hosts.yml 16 | inventory-test/group_vars/all.yml 17 | roles/monitoring/files/test.py 18 | inventory/group_vars/rpc.yaml 19 | -------------------------------------------------------------------------------- /CNAME: -------------------------------------------------------------------------------- 1 | sv-manager.thevalidators.io -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Solana Validator Manager 2 | 3 | ### Automatically bootstrap a Solana validator node, optimize its performance, and connect the node to a monitoring dashboard 4 | 5 | [Solana](https://solana.com/) is a fast, secure, and censorship-resistant blockchain providing open infrastructure necessary for global adoption. 6 | 7 | In order to run, the Solana blockchain requires a decentralized network comprising computing resources to validate transactions as well as storage for ledger redundancy. 8 | 9 | The computer resources are provided by validators who need to maintain high-performance Linux nodes. 10 | 11 | There are now two Solana clusters, [Mainnet-Beta](https://explorer.solana.com/) and [Testnet](https://explorer.solana.com/?cluster=testnet). 12 | 13 | The Mainnet-Beta cluster is maintained by ~700 validators, and the Testnet cluster by ~1700 more validators. 14 | 15 | Most of the people running these just bootstrap their nodes manually, referring to the Solana docs or similar community guides. Apparently, there are no 2 identical setups across these 2400 validators. 16 | 17 | As a result, it is virtually impossible to support validators having issues with their nodes and/or help them improve their node, thus contributing to the overall cluster performance. 18 | 19 | What we would like to do is provide a toolkit to help validators bootstrap and maintain their nodes in a uniform, consistent way. 20 | 21 | The Ansible scripts we have created for this purpose are a compilation of best practices and community guidelines. 22 | 23 | Please use them, enjoy them, and improve them. 24 | 25 | ### Quick Install 26 | 27 | * Log in to your server 28 | * Create the key pair file (you can also upload it via scp if you prefer): 29 | ````shell 30 | nano ~/validator-keypair.json 31 | ```` 32 | Paste your key pair, save the file (ctrl-O) and exit (ctrl-X). 33 | 34 | 35 | If you have a *vote account* key pair, create the key pair file (or upload it via scp): 36 | ````shell 37 | nano ~/vote-account-keypair.json 38 | ```` 39 | Paste your key pair, save the file (ctrl-O) and exit (ctrl-X). 40 | * Run this command… 41 | 42 | ````shell 43 | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/mfactory-lab/sv-manager/latest/install/install_validator.sh)" 44 | ```` 45 | 46 | …and follow the wizard’s instructions (__enter your own Node name!__): 47 | 48 | 49 | 50 | That's it, you are all set! 51 | 52 | ### How to update validator 53 | 54 | ````shell 55 | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/mfactory-lab/sv-manager/latest/install/update_test_validator_version.sh)" --version 1.14.2 56 | ```` 57 | 58 | ### how to update monitoring 59 | 60 | ````shell 61 | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/mfactory-lab/sv-manager/latest/install/update_monitoring.sh)" 62 | ```` 63 | 64 | 65 | ### If you want more control over the configuration of your node, please refer to the [advanced technical specifications](docs/advanced.md) 66 | 67 | 68 | ## Useful links 69 | 70 | * [Solana](https://solana.com/) 71 | * [Monitoring Dashboard](https://solana.thevalidators.io/) 72 | * [Validator docs](https://docs.solana.com/running-validator) 73 | 74 | ## How you can support this project 75 | 76 | - Join our Telegram community [t.me/thevalidators](https://t.me/thevalidators) 77 | - Fork, improve, and promote 78 | - Stake with [Joogh Validator](https://solanabeach.io/validator/DPmsofVJ1UMRZADgwYAHotJnazMwohHzRHSoomL6Qcao) 79 | - Donate Sol to [Joogh Validator Identity Account](https://joogh.io) on Solana: 8yjHdsCgx3bp2zEwGiWSMgwpFaCSzfYAHT1vk7KJBqhN 80 | - Donate BTC: bc1q9vkmfpmk77j2kcsdy2slnv6ld4ahg2g5guysvy 81 | 82 | ### [Powered by mFactory Team](https://mfactory.tech) 83 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-hacker -------------------------------------------------------------------------------- /ansible.cfg: -------------------------------------------------------------------------------- 1 | [defaults] 2 | callback_whitelist = debug 3 | stdout_callback = debug 4 | roles_path = ./roles 5 | inventory = ./inventory 6 | 7 | host_key_checking = False 8 | -------------------------------------------------------------------------------- /docs/advanced.md: -------------------------------------------------------------------------------- 1 | # Solana Validation Manager – Advanced Manual 2 | 3 | - [Node requirements](#node-requirements) 4 | - [What does it do with your Validator Node exactly](#what-does-it-do-with-your-validator-node-exactly) 5 | - [Configure Ubuntu](#configure-ubuntu) 6 | - [Bootstrap Solana cli](#bootstrap-solana-cli) 7 | - [Configure node monitoring](#configure-node-monitoring) 8 | - [How to install Ansible](#how-to-install-ansible) 9 | - [on MacOS](#on-macos) 10 | - [on Ubuntu](#on-ubuntu) 11 | - [on Windows](#on-windows) 12 | - [How to configure Solana Validator Manager](#how-to-configure-solana-validator-manager) 13 | - [How to use](#how-to-use) 14 | - [from local machine](#from-local-machine) 15 | - [directly from Validator Node](#from-validator-node) 16 | - [via docker](#via-docker) 17 | - [Which functions are supported](#which-functions-are-supported) 18 | - [install](#install-validator-node) 19 | - [update Solana Version](#update-solana-version) 20 | - [install or update monitoring](#install-or-update-monitoring) 21 | - [how to install monitoring with ansible](#how-to-install-monitoring-with-ansible) 22 | - [how to install monitoring manually](#how-to-install-monitoring-manually) 23 | - [migrate your current setup to supported by sv-manager](#migrate-your-current-setup-to-supported-by-sv-manager) 24 | - [how to migrate your setup semi automatically](#how-to-migrate-your-setup-semi-automatically) 25 | - [how to migrate your setup manually](#how-to-migrate-your-setup-manually) 26 | [Useful links](#useful-links) 27 | - [How can you support this project](#how-can-you-support-this-project) 28 | - [History](history.md) 29 | - [Roadmap](roadmap.md) 30 | 31 | 32 | ## Node requirements 33 | 34 | * We have tested it only with Ubuntu 20.04, but every linux distro with apt and systemd should be supported. 35 | * Support for other Linux-Distributives will be implemented soon. 36 | * Check Solana [Validator requirements](https://docs.solana.com/running-validator/validator-reqs) 37 | 38 | ## What does it do with your Validator Node exactly 39 | 40 | ### Configure Ubuntu 41 | 42 | [Ubuntu role](../roles/configure_ubuntu) 43 | 44 | that role configures your ubuntu node to be more performant and stable with validation 45 | 46 | 1. Create ansible user 47 | 2. Create solana user 48 | 3. Create swap file 49 | 4. Create ram disk for accounts db 50 | 5. Set cpu governor to performance 51 | 6. Configure firewall 52 | 53 | ### Bootstrap Solana cli 54 | 55 | [Solana cli role](../roles/solana_cli) 56 | 57 | that role installs or updates solana cli 58 | 59 | ### Configure node monitoring 60 | 61 | - [monitoring](../roles/monitoring) 62 | 63 | that role configures sending of validator and node metrics to our [grafana dashboard](https://solana.thevalidators.io) 64 | 65 | ## How to install Ansible 66 | 67 | ### On macOS 68 | 69 | brew update; brew install ansible 70 | 71 | ### On Ubuntu 72 | 73 | apt-get update; apt-get install ansible 74 | 75 | ### On Windows 76 | 77 | unfortunately ansible is not directly supported on Windows, we are working on a docker image 78 | which will directly provide a support for Solana Validator Manager, until that you can use 79 | [ansible docker image](https://hub.docker.com/r/ansible/ansible) on your own. 80 | 81 | ## How to configure Solana Validator Manager 82 | 83 | *!only testnet configuration is supported now!* 84 | 85 | 1. clone git repository 86 | git clone ... 87 | 1. copy or rename [inventory_example directory](../inventory_example) to directory named *inventory* 88 | 2. add to your inventory/hosts.yaml 89 | 90 | * validator name 91 | * ip-address of your validator node 92 | 93 | ````yaml 94 | all: 95 | children: 96 | local: 97 | vars: 98 | become: false 99 | ansible_connection: local 100 | ansible_python_interpreter: "{{ ansible_playbook_python }}" 101 | hosts: 102 | localhost 103 | remote: 104 | vars: 105 | ansible_user: root 106 | children: 107 | validator: 108 | vars: 109 | validator_name: <- HERE 110 | # keybase_username: "" 111 | # validator_homepage: "" 112 | upload_validator_keys: False <- Set it to True if you want to upload your keys 113 | # secrets_path: /home/solana/.secrets 114 | # set_validator_info: True 115 | # service_user: solana 116 | # ledger_path: /home/solana/ledger 117 | # lvm_enabled: False 118 | # lvm_vg: vg00 119 | # solana_validator_service: restarted <- Set it to stopped if you want to check your setup 120 | # swap_file_size_gb: 64 121 | # ramdisk_size_gb: 64 122 | # cluster_environment: testnet 123 | # cluster_rpc_address: https://api.testnet.solana.com 124 | hosts: 125 | 0.0.0.0 <- HERE 126 | ```` 127 | 128 | in case you have already identity keys (the most of you will have at least a validator identity key) 129 | 130 | 4. copy your validator identity key, if any, into .secrets/{{YOUR VALIDATOR NAME}}/solana 131 | 5. copy your vote account identity key, if any, into .secrets/{{YOUR VALIDATOR NAME}}/solana 132 | 6. set upload_validator_keys to True 133 | 134 | ## How to use 135 | 136 | ### from local machine 137 | 138 | 1. make sure you have access to you validator node as a root user via [ssh-key](https://www.cyberciti.biz/faq/ubuntu-18-04-setup-ssh-public-key-authentication/) 139 | 2. configure hosts.yaml 140 | 3. start playbook 141 | 142 | ansible-playbook install_validator.yaml -v 143 | 144 | if you run an ubuntu server with more or less standard configuration than in few minutes 145 | your validator node should be up, and you can observe your metrics on [our metrics dashboard](https://solana.thevalidators.io) 146 | 147 | ### from Validator Node 148 | 149 | coming soon! 150 | 151 | ## Which functions are supported 152 | 153 | ### Install Validator Node 154 | 155 | ansible-playbook pb_install_validator.yaml -v 156 | 157 | ### Update Solana Version 158 | 159 | * bump solana version in group_vars/all.yaml 160 | 161 | ```yaml 162 | ... 163 | validator: 164 | version: v1.6.7 <- set new version here 165 | ... 166 | ``` 167 | 168 | * start ansible playbook 169 | 170 | ansible-playbook pb_update_validator.yaml -v 171 | 172 | ### Install or update Monitoring 173 | 174 | * if you just want to use monitoring 175 | * or want update monitoring library 176 | 177 | #### How to install monitoring with ansible 178 | 179 | ansible-playbook pb_install_monitoring.yaml -v 180 | 181 | #### How to install monitoring manually 182 | 183 | From server command line, user root, paste the whole command and run it: 184 | 185 | /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/mfactory-lab/sv-manager/latest/install/install_monitoring.sh)" 186 | 187 | ### Migrate your current setup to supported by sv-manager 188 | 189 | #### How to migrate your setup semi-automatically 190 | 191 | coming soon 192 | 193 | #### How to migrate your setup manually 194 | 195 | coming soon 196 | 197 | -------------------------------------------------------------------------------- /docs/history.md: -------------------------------------------------------------------------------- 1 | # 0.1 2 | - first release 3 | -------------------------------------------------------------------------------- /docs/launch.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mfactory-lab/sv-manager/718946c5b1d6d09bdbd1833117a90a487bb68ab1/docs/launch.gif -------------------------------------------------------------------------------- /docs/pitch.md: -------------------------------------------------------------------------------- 1 | # Pitch 2 | 3 | ## Problem 4 | 5 | Solana is a high speed blockchain network which requires fast, stable validator nodes, with high decentralisation across the data centers in the world. 6 | At the time of writing, there are currently about 3500 validators on the Testnet and about 1400 on the Mainnet. The majority of them are set up using various guides and information available on the Internet. 7 | These are ad-hoc, unique installations with different software versions, settings, configurations and user profiles. 8 | Many people running validator nodes have little to zero knowledge about system administration, troubleshooting and monitoring. 9 | High (knowledge) entry barrier attracts people to start their nodes only in a small count of data centers mentioned in guides and YouTube videos. 10 | These facts lead to less cluster performance, stability, resilience and decentralisation. On the other hand, it is hard to impossible to provide any support in case of a validator's failures or bad performance. 11 | 12 | ## Solution 13 | 14 | A toolkit allowing to set up a validator node and monitoring in one-click manner with consistent common settings for everyone by providing an interactive shell script. It is based on an Infrastructure-As-Code solution and includes an Ansible pack for experienced users to use as references and best practice documents. 15 | 16 | The monitoring can be installed separately using a shell script. 17 | 18 | An experienced team stands behind the project for further development and for multilingual user support. 19 | 20 | Features already implemented: 21 | - Node bootstrapping 22 | - Solana version upgrade 23 | - Various node and cluster monitoring dashboards 24 | - Simple alerts (via Telegram bot and other channels) 25 | 26 | Currently being developed: 27 | - Standalone solution enabling anyone to launch a private monitoring and alerting system, including dashboard and all other sv-manager features 28 | 29 | Future plans: 30 | - Advanced alerting system 31 | - Disaster recovery 32 | - Mobile cockpit for alerts, notifications, and node control on the go 33 | - Open API enabling anyone to use the monitoring data 34 | - Automated node startup in case of cluster restart 35 | 36 | Our solution increases network decentralization by lowering the barrier to entry: the toolkit enables a much larger audience to set up and run a validator node with higher efficiency and resiliency of the network. 37 | 38 | With our SV (Solana Validator) Manager (https://sv-manager.thevalidators.io/) we currently implemented a minimum viable product (MVP) which is actively used by more than 100 validators. 39 | 40 | ## Competition 41 | 42 | - Setup: Various guidelines offered on GitHub, YouTube, a number of blogs. 43 | - Monitoring: validators.app, stakeconomy 44 | - *Nobody provides a full package* 45 | 46 | ## Team 47 | 48 | 1. Alexander Ray (https://www.linkedin.com/in/alex-a-ray/) 49 | 2. Vladimir Polyatskin (https://www.linkedin.com/in/vladimir-polyatskin-839923154/) 50 | 51 | ## Why You? 52 | 53 | A multi-faceted team covering a wide range of important skills such as 54 | engineering, development, monitoring, operations and security in both application and infrastructure domains, as well 20+ years experience in these areas. 55 | 56 | ## Previous Work 57 | 58 | - globar: https://www.youtube.com/watch?v=PFpokgVh0gg 59 | - mwall: https://www.youtube.com/watch?v=oBLZtNEQ-zE 60 | - profiter: https://profiter.mfactory.de/ 61 | - pltool: https://pl-tool.com/ 62 | - github: https://github.com/mfactory-lab 63 | - JPool: https://jpool.one/ 64 | 65 | ## Amount 66 | 67 | Depending on the available funding, we would be able to dedicate a varying amount of our time to this project. Better funding means we would be able to fully achieve the goals listed above and do it faster, while also offering more support for the Solana Validator community. 68 | 69 | We see the SV-Manager as a long-term project that needs to be constantly developed and enhanced together with Solana, and we see its future as being most successful in close cooperation with the Solana Foundation 70 | -------------------------------------------------------------------------------- /docs/roadmap.md: -------------------------------------------------------------------------------- 1 | * improve monitoring 2 | * alerting 3 | ... 4 | -------------------------------------------------------------------------------- /docs/wizard.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mfactory-lab/sv-manager/718946c5b1d6d09bdbd1833117a90a487bb68ab1/docs/wizard.gif -------------------------------------------------------------------------------- /install/debug.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | echo "Downloading Solana validator manager" 5 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$1.zip" 6 | echo "starting $cmd" 7 | curl -fsSL "$cmd" --output sv_manager.zip 8 | echo "Unpacking" 9 | unzip ./sv_manager.zip -d . 10 | 11 | mv sv-manager* sv_manager 12 | rm ./sv_manager.zip 13 | cd ./sv_manager || exit 14 | cp -r ./inventory_example ./inventory 15 | 16 | ansible-playbook --connection=local --inventory ./inventory -l local playbooks/pb_debug.yaml -------------------------------------------------------------------------------- /install/install_agave_validator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | echo "###################### WARNING!!! ######################" 5 | echo "### This script will bootstrap a validator node ###" 6 | echo "### for the Solana Testnet cluster, and connect ###" 7 | echo "### it to the monitoring dashboard ###" 8 | echo "### at solana.thevalidators.io ###" 9 | echo "########################################################" 10 | 11 | install_validator () { 12 | 13 | echo "### Which type of validator you want to set up? ###" 14 | select cluster in "mainnet-beta" "testnet"; do 15 | case $cluster in 16 | mainnet-beta ) inventory="mainnet.yaml"; break;; 17 | testnet ) inventory="testnet.yaml"; break;; 18 | esac 19 | done 20 | 21 | echo "Please enter a name for your validator node: " 22 | read VALIDATOR_NAME 23 | read -e -p "Please enter the full path to your validator key pair file: " -i "/root/" PATH_TO_VALIDATOR_KEYS 24 | 25 | if [ ! -f "$PATH_TO_VALIDATOR_KEYS/validator-keypair.json" ] 26 | then 27 | echo "OOPS! Key $PATH_TO_VALIDATOR_KEYS/validator-keypair.json not found. Please verify and run the script again" 28 | exit 29 | fi 30 | 31 | if [ ! -f "$PATH_TO_VALIDATOR_KEYS/vote-account-keypair.json" ] ## && [ "$inventory" = "mainnet.yaml" ] 32 | then 33 | echo "OOPS! Key $PATH_TO_VALIDATOR_KEYS/vote-account-keypair.json not found. Please verify and run the script again. For security reasons we do not create any keys for mainnet." 34 | exit 35 | fi 36 | 37 | read -e -p "Enter new RAM drive size, GB (recommended size: 200GB):" -i "200" RAM_DISK_SIZE 38 | read -e -p "Enter new server swap size, GB (recommended size: equal to server RAM): " -i "64" SWAP_SIZE 39 | 40 | rm -rf sv_manager/ 41 | 42 | if [[ $(which apt | wc -l) -gt 0 ]] 43 | then 44 | pkg_manager=apt 45 | elif [[ $(which yum | wc -l) -gt 0 ]] 46 | then 47 | pkg_manager=yum 48 | fi 49 | 50 | echo "Updating packages..." 51 | $pkg_manager update 52 | echo "Installing ansible, curl, unzip..." 53 | $pkg_manager install ansible curl unzip --yes 54 | 55 | ansible-galaxy collection install ansible.posix 56 | ansible-galaxy collection install community.general 57 | 58 | echo "Downloading Solana validator manager version $sv_manager_version" 59 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$sv_manager_version.zip" 60 | echo "starting $cmd" 61 | curl -fsSL "$cmd" --output sv_manager.zip 62 | echo "Unpacking" 63 | unzip ./sv_manager.zip -d . 64 | 65 | mv sv-manager* sv_manager 66 | rm ./sv_manager.zip 67 | cd ./sv_manager || exit 68 | cp -r ./inventory_example ./inventory 69 | 70 | # shellcheck disable=SC2154 71 | #echo "pwd: $(pwd)" 72 | #ls -lah ./ 73 | 74 | if [ ! -z $solana_version ] 75 | then 76 | SOLANA_VERSION="--extra-vars {\"agave_version\":\"$agave_version\"}" 77 | fi 78 | if [ ! -z $extra_vars ] 79 | then 80 | EXTRA_INSTALL_VARS="--extra-vars $extra_vars" 81 | fi 82 | if [ ! -z $tags ] 83 | then 84 | TAGS="--tags [$tags]" 85 | fi 86 | 87 | if [ ! -z $skip_tags ] 88 | then 89 | SKIP_TAGS="--skip-tags $skip_tags" 90 | fi 91 | 92 | ansible-playbook --connection=local --inventory ./inventory/$inventory --limit localhost playbooks/pb_config.yaml --extra-vars "{ \ 93 | 'validator_name':'$VALIDATOR_NAME', \ 94 | 'local_secrets_path': '$PATH_TO_VALIDATOR_KEYS', \ 95 | 'swap_file_size_gb': $SWAP_SIZE, \ 96 | 'ramdisk_size_gb': $RAM_DISK_SIZE, \ 97 | }" $SOLANA_VERSION $EXTRA_INSTALL_VARS $TAGS $SKIP_TAGS 98 | 99 | ansible-playbook --connection=local --inventory ./inventory/$inventory --limit localhost playbooks/pb_install_agave_validator.yaml --extra-vars "@/etc/sv_manager/sv_manager.conf" $SOLANA_VERSION $EXTRA_INSTALL_VARS $TAGS $SKIP_TAGS 100 | 101 | echo "### 'Uninstall ansible ###" 102 | 103 | $pkg_manager remove ansible --yes 104 | if [ "$inventory" = "mainnet.yaml" ] 105 | then 106 | echo "WARNING: solana is ready to go. But you must start it by the hand. Use \"systemctl start solana-validator\" command." 107 | fi 108 | 109 | 110 | echo "### Check your dashboard: https://solana.thevalidators.io/d/e-8yEOXMwerfwe/solana-monitoring?&var-server=$VALIDATOR_NAME" 111 | 112 | } 113 | 114 | 115 | while [ $# -gt 0 ]; do 116 | 117 | if [[ $1 == *"--"* ]]; then 118 | param="${1/--/}" 119 | declare ${param}="$2" 120 | # echo $1 $2 // Optional to see the parameter:value result 121 | fi 122 | 123 | shift 124 | done 125 | 126 | sv_manager_version=${sv_manager_version:-latest} 127 | 128 | echo "installing sv manager version $sv_manager_version" 129 | 130 | echo "This script will bootstrap a Solana validator node. Proceed?" 131 | select yn in "Yes" "No"; do 132 | case $yn in 133 | Yes ) install_validator "$sv_manager_version" "$extra_vars" "$solana_version" "$tags" "$skip_tags"; break;; 134 | No ) echo "Aborting install. No changes will be made."; exit;; 135 | esac 136 | done 137 | -------------------------------------------------------------------------------- /install/install_monitoring.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | echo "###################### WARNING!!! ######################" 5 | echo "### This script will install and/or reconfigure ###" 6 | echo "### telegraf and point it to solana.thevalidators.io ###" 7 | echo "########################################################" 8 | 9 | install_monitoring () { 10 | 11 | echo "### Which cluster you wnat to monitor? ###" 12 | select cluster in "mainnet-beta" "testnet"; do 13 | case $cluster in 14 | mainnet-beta ) inventory="mainnet.yaml"; break;; 15 | testnet ) inventory="testnet.yaml"; break;; 16 | esac 17 | done 18 | 19 | 20 | echo "### Please type your validator name: " 21 | read VALIDATOR_NAME 22 | echo "### Please type the full path to your validator keys: " 23 | read PATH_TO_VALIDATOR_KEYS 24 | 25 | if [ ! -f "$PATH_TO_VALIDATOR_KEYS/validator-keypair.json" ] 26 | then 27 | echo "key $PATH_TO_VALIDATOR_KEYS/validator-keypair.json not found. Pleas verify and run the script again" 28 | exit 29 | fi 30 | 31 | read -e -p "### Please tell which user is running validator: " SOLANA_USER 32 | cd 33 | rm -rf sv_manager/ 34 | 35 | if [[ $(which apt | wc -l) -gt 0 ]] 36 | then 37 | pkg_manager=apt 38 | elif [[ $(which yum | wc -l) -gt 0 ]] 39 | then 40 | pkg_manager=yum 41 | fi 42 | 43 | echo "### Update packages... ###" 44 | $pkg_manager update 45 | echo "### Install ansible, curl, unzip... ###" 46 | $pkg_manager install ansible curl unzip --yes 47 | 48 | # fix for eventually hanging of pip 49 | export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring 50 | 51 | ansible-galaxy collection install ansible.posix 52 | ansible-galaxy collection install community.general 53 | 54 | echo "### Download Solana validator manager" 55 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$1.zip" 56 | echo "starting $cmd" 57 | curl -fsSL "$cmd" --output sv_manager.zip 58 | echo "### Unpack Solana validator manager ###" 59 | unzip ./sv_manager.zip -d . 60 | 61 | mv sv-manager* sv_manager 62 | rm ./sv_manager.zip 63 | cd ./sv_manager || exit 64 | cp -r ./inventory_example ./inventory 65 | 66 | #echo $(pwd) 67 | ansible-playbook --connection=local --inventory ./inventory/$inventory --limit localhost playbooks/pb_config.yaml --extra-vars "{ \ 68 | 'solana_user': '$SOLANA_USER', \ 69 | 'validator_name':'$VALIDATOR_NAME', \ 70 | 'local_secrets_path': '$PATH_TO_VALIDATOR_KEYS' \ 71 | }" 72 | 73 | ansible-playbook --connection=local --inventory ./inventory/$inventory --limit localhost playbooks/pb_install_monitoring.yaml --extra-vars "@/etc/sv_manager/sv_manager.conf" 74 | 75 | echo "### Cleanup install folder ###" 76 | cd .. 77 | rm -r ./sv_manager 78 | echo "### Cleanup install folder done ###" 79 | echo "### Check your dashboard: https://solana.thevalidators.io/d/e-8yEOXMwerfwe/solana-monitoring?&var-server="$VALIDATOR_NAME 80 | 81 | echo Do you want to UNinstall ansible? 82 | select yn in "Yes" "No"; do 83 | case $yn in 84 | Yes ) $pkg_manager remove ansible --yes; break;; 85 | No ) echo "### Okay, ansible is still installed on this system. ###"; break;; 86 | esac 87 | done 88 | 89 | } 90 | 91 | echo Do you want to install monitoring? 92 | select yn in "Yes" "No"; do 93 | case $yn in 94 | Yes ) install_monitoring "${1:-latest}"; break;; 95 | No ) echo "### Aborting install. No changes are made on the system."; exit;; 96 | esac 97 | done 98 | -------------------------------------------------------------------------------- /install/install_rpc_node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | echo "###################### WARNING!!! ######################" 5 | echo "### This script will bootstrap an RPC node ###" 6 | echo "### for the Solana blockchain, and connect ###" 7 | echo "### it to the monitoring dashboard ###" 8 | echo "### at solana.thevalidators.io ###" 9 | echo "########################################################" 10 | 11 | install_rpc () { 12 | 13 | echo "### Please choose the cluster: ###" 14 | select cluster in "mainnet-beta" "testnet"; do 15 | case $cluster in 16 | mainnet-beta ) inventory="mainnet.yaml"; break;; 17 | testnet ) inventory="testnet.yaml"; break;; 18 | esac 19 | done 20 | 21 | echo "Please enter a name for your RPC node: " 22 | read VALIDATOR_NAME 23 | read -e -p "Please enter the full path to your validator key pair file or leave it blank, then the keys will be created: " -i "" PATH_TO_VALIDATOR_KEYS 24 | 25 | 26 | read -e -p "Enter new RAM drive size, GB (recommended size: server RAM minus 16GB):" -i "48" RAM_DISK_SIZE 27 | read -e -p "Enter new server swap size, GB (recommended size: equal to server RAM): " -i "64" SWAP_SIZE 28 | 29 | rm -rf sv_manager/ 30 | 31 | if [[ $(which apt | wc -l) -gt 0 ]] 32 | then 33 | pkg_manager=apt 34 | elif [[ $(which yum | wc -l) -gt 0 ]] 35 | then 36 | pkg_manager=yum 37 | fi 38 | 39 | echo "Updating packages..." 40 | $pkg_manager update 41 | echo "Installing ansible, curl, unzip..." 42 | $pkg_manager install ansible curl unzip --yes 43 | 44 | ansible-galaxy collection install ansible.posix 45 | ansible-galaxy collection install community.general 46 | 47 | echo "Downloading Solana validator manager version $sv_manager_version" 48 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$sv_manager_version.zip" 49 | echo "starting $cmd" 50 | curl -fsSL "$cmd" --output sv_manager.zip 51 | echo "Unpacking" 52 | unzip ./sv_manager.zip -d . 53 | 54 | mv sv-manager* sv_manager 55 | rm ./sv_manager.zip 56 | cd ./sv_manager || exit 57 | cp -r ./inventory_example ./inventory 58 | 59 | # shellcheck disable=SC2154 60 | #echo "pwd: $(pwd)" 61 | #ls -lah ./ 62 | 63 | ansible-playbook --connection=local --inventory ./inventory/$inventory --limit localhost_rpc playbooks/pb_config.yaml --extra-vars "{ \ 64 | 'validator_name':'$VALIDATOR_NAME', \ 65 | 'local_secrets_path': '$PATH_TO_VALIDATOR_KEYS', \ 66 | 'swap_file_size_gb': $SWAP_SIZE, \ 67 | 'ramdisk_size_gb': $RAM_DISK_SIZE, \ 68 | 'fail_if_no_validator_keypair: False' 69 | }" 70 | 71 | if [ ! -z $solana_version ] 72 | then 73 | SOLANA_VERSION="--extra-vars {\"solana_version\":\"$solana_version\"}" 74 | fi 75 | if [ ! -z $extra_vars ] 76 | then 77 | EXTRA_INSTALL_VARS="--extra-vars {$extra_vars}" 78 | fi 79 | if [ ! -z $tags ] 80 | then 81 | TAGS="--tags {$tags}" 82 | fi 83 | 84 | ansible-playbook --connection=local --inventory ./inventory/$inventory --limit localhost_rpc playbooks/pb_install_validator.yaml --extra-vars "@/etc/sv_manager/sv_manager.conf" $SOLANA_VERSION $EXTRA_INSTALL_VARS $TAGS 85 | 86 | echo "### 'Uninstall ansible ###" 87 | 88 | $pkg_manager remove ansible --yes 89 | 90 | echo "### Check your dashboard: https://solana.thevalidators.io/d/e-8yEOXMwerfwe/solana-monitoring?&var-server=$VALIDATOR_NAME" 91 | 92 | } 93 | 94 | 95 | while [ $# -gt 0 ]; do 96 | 97 | if [[ $1 == *"--"* ]]; then 98 | param="${1/--/}" 99 | declare ${param}="$2" 100 | #echo $1 $2 // Optional to see the parameter:value result 101 | fi 102 | 103 | shift 104 | done 105 | 106 | sv_manager_version=${sv_manager_version:-latest} 107 | 108 | echo "installing sv manager version $sv_manager_version" 109 | 110 | echo "This script will bootstrap a Solana RPC node. Proceed?" 111 | select yn in "Yes" "No"; do 112 | case $yn in 113 | Yes ) install_rpc "$sv_manager_version" "$extra_vars" "$solana_version" "$tags"; break;; 114 | No ) echo "Aborting install. No changes will be made."; exit;; 115 | esac 116 | done 117 | -------------------------------------------------------------------------------- /install/install_update_sys_tuner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | echo "###################### WARNING!!! ######################" 5 | echo "### This script will install the Solana Sys Tuner ###" 6 | echo "### for the Solana Validator. ###" 7 | echo "########################################################" 8 | 9 | install_validator () { 10 | 11 | rm -rf sv_manager/ 12 | 13 | if [[ $(which apt | wc -l) -gt 0 ]] 14 | then 15 | pkg_manager=apt 16 | elif [[ $(which yum | wc -l) -gt 0 ]] 17 | then 18 | pkg_manager=yum 19 | fi 20 | 21 | echo "Updating packages..." 22 | $pkg_manager update 23 | echo "Installing ansible, curl, unzip..." 24 | $pkg_manager install ansible curl unzip --yes 25 | 26 | ansible-galaxy collection install ansible.posix 27 | ansible-galaxy collection install community.general 28 | 29 | echo "Downloading Solana validator manager version $sv_manager_version" 30 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$sv_manager_version.zip" 31 | echo "starting $cmd" 32 | curl -fsSL "$cmd" --output sv_manager.zip 33 | echo "Unpacking" 34 | unzip ./sv_manager.zip -d . 35 | 36 | mv sv-manager* sv_manager 37 | rm ./sv_manager.zip 38 | cd ./sv_manager || exit 39 | cp -r ./inventory_example ./inventory 40 | 41 | # shellcheck disable=SC2154 42 | #echo "pwd: $(pwd)" 43 | #ls -lah ./ 44 | 45 | 46 | ansible-playbook --connection=local --inventory ./inventory/mainnet.yaml --limit localhost playbooks/pb_install_validator.yaml --tags validator.service.sys-tuner 47 | 48 | echo "### 'Uninstall ansible ###" 49 | 50 | $pkg_manager remove ansible --yes 51 | 52 | echo "### Solana Sys Tuner Servie installed. You can check the status with 'systemctl status solana-sys-tuner' ###" 53 | 54 | 55 | } 56 | 57 | 58 | while [ $# -gt 0 ]; do 59 | 60 | if [[ $1 == *"--"* ]]; then 61 | param="${1/--/}" 62 | declare ${param}="$2" 63 | # echo $1 $2 // Optional to see the parameter:value result 64 | fi 65 | 66 | shift 67 | done 68 | 69 | sv_manager_version=${sv_manager_version:-latest} 70 | 71 | echo "installing sv manager version $sv_manager_version" 72 | 73 | echo "This script will setup the Solana Sys Tuner Service with default parameters. Proceed?" 74 | select yn in "Yes" "No"; do 75 | case $yn in 76 | Yes ) install_validator "$sv_manager_version" "$extra_vars" "$solana_version" "$tags" "$skip_tags"; break;; 77 | No ) echo "Aborting install. No changes will be made."; exit;; 78 | esac 79 | done 80 | -------------------------------------------------------------------------------- /install/install_validator.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | echo "###################### WARNING!!! ######################" 5 | echo "### This script will bootstrap a validator node ###" 6 | echo "### for the Solana Testnet cluster, and connect ###" 7 | echo "### it to the monitoring dashboard ###" 8 | echo "### at solana.thevalidators.io ###" 9 | echo "########################################################" 10 | 11 | install_validator () { 12 | 13 | echo "### Which type of validator you want to set up? ###" 14 | select cluster in "mainnet-beta" "testnet"; do 15 | case $cluster in 16 | mainnet-beta ) inventory="mainnet.yaml"; break;; 17 | testnet ) inventory="testnet.yaml"; break;; 18 | esac 19 | done 20 | 21 | echo "Please enter a name for your validator node: " 22 | read VALIDATOR_NAME 23 | read -e -p "Please enter the full path to your validator key pair file: " -i "/root/" PATH_TO_VALIDATOR_KEYS 24 | 25 | if [ ! -f "$PATH_TO_VALIDATOR_KEYS/validator-keypair.json" ] 26 | then 27 | echo "OOPS! Key $PATH_TO_VALIDATOR_KEYS/validator-keypair.json not found. Please verify and run the script again" 28 | exit 29 | fi 30 | 31 | if [ ! -f "$PATH_TO_VALIDATOR_KEYS/vote-account-keypair.json" ] ## && [ "$inventory" = "mainnet.yaml" ] 32 | then 33 | echo "OOPS! Key $PATH_TO_VALIDATOR_KEYS/vote-account-keypair.json not found. Please verify and run the script again. For security reasons we do not create any keys for mainnet." 34 | exit 35 | fi 36 | 37 | read -e -p "Enter new RAM drive size, GB (recommended size: 200GB):" -i "200" RAM_DISK_SIZE 38 | read -e -p "Enter new server swap size, GB (recommended size: equal to server RAM): " -i "64" SWAP_SIZE 39 | 40 | rm -rf sv_manager/ 41 | 42 | if [[ $(which apt | wc -l) -gt 0 ]] 43 | then 44 | pkg_manager=apt 45 | elif [[ $(which yum | wc -l) -gt 0 ]] 46 | then 47 | pkg_manager=yum 48 | fi 49 | 50 | echo "Updating packages..." 51 | $pkg_manager update 52 | echo "Installing ansible, curl, unzip..." 53 | $pkg_manager install ansible curl unzip --yes 54 | 55 | ansible-galaxy collection install ansible.posix 56 | ansible-galaxy collection install community.general 57 | 58 | echo "Downloading Solana validator manager version $sv_manager_version" 59 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$sv_manager_version.zip" 60 | echo "starting $cmd" 61 | curl -fsSL "$cmd" --output sv_manager.zip 62 | echo "Unpacking" 63 | unzip ./sv_manager.zip -d . 64 | 65 | mv sv-manager* sv_manager 66 | rm ./sv_manager.zip 67 | cd ./sv_manager || exit 68 | cp -r ./inventory_example ./inventory 69 | 70 | # shellcheck disable=SC2154 71 | #echo "pwd: $(pwd)" 72 | #ls -lah ./ 73 | 74 | if [ ! -z $solana_version ] 75 | then 76 | SOLANA_VERSION="--extra-vars {\"solana_version\":\"$solana_version\"}" 77 | fi 78 | if [ ! -z $extra_vars ] 79 | then 80 | EXTRA_INSTALL_VARS="--extra-vars $extra_vars" 81 | fi 82 | if [ ! -z $tags ] 83 | then 84 | TAGS="--tags [$tags]" 85 | fi 86 | 87 | if [ ! -z $skip_tags ] 88 | then 89 | SKIP_TAGS="--skip-tags $skip_tags" 90 | fi 91 | 92 | ansible-playbook --connection=local --inventory ./inventory/$inventory --limit localhost playbooks/pb_config.yaml --extra-vars "{ \ 93 | 'validator_name':'$VALIDATOR_NAME', \ 94 | 'local_secrets_path': '$PATH_TO_VALIDATOR_KEYS', \ 95 | 'swap_file_size_gb': $SWAP_SIZE, \ 96 | 'ramdisk_size_gb': $RAM_DISK_SIZE, \ 97 | }" $SOLANA_VERSION $EXTRA_INSTALL_VARS $TAGS $SKIP_TAGS 98 | 99 | ansible-playbook --connection=local --inventory ./inventory/$inventory --limit localhost playbooks/pb_install_validator.yaml --extra-vars "@/etc/sv_manager/sv_manager.conf" $SOLANA_VERSION $EXTRA_INSTALL_VARS $TAGS $SKIP_TAGS 100 | 101 | echo "### 'Uninstall ansible ###" 102 | 103 | $pkg_manager remove ansible --yes 104 | if [ "$inventory" = "mainnet.yaml" ] 105 | then 106 | echo "WARNING: solana is ready to go. But you must start it by the hand. Use \"systemctl start solana-validator\" command." 107 | fi 108 | 109 | 110 | echo "### Check your dashboard: https://solana.thevalidators.io/d/e-8yEOXMwerfwe/solana-monitoring?&var-server=$VALIDATOR_NAME" 111 | 112 | } 113 | 114 | 115 | while [ $# -gt 0 ]; do 116 | 117 | if [[ $1 == *"--"* ]]; then 118 | param="${1/--/}" 119 | declare ${param}="$2" 120 | # echo $1 $2 // Optional to see the parameter:value result 121 | fi 122 | 123 | shift 124 | done 125 | 126 | sv_manager_version=${sv_manager_version:-latest} 127 | 128 | echo "installing sv manager version $sv_manager_version" 129 | 130 | echo "This script will bootstrap a Solana validator node. Proceed?" 131 | select yn in "Yes" "No"; do 132 | case $yn in 133 | Yes ) install_validator "$sv_manager_version" "$extra_vars" "$solana_version" "$tags" "$skip_tags"; break;; 134 | No ) echo "Aborting install. No changes will be made."; exit;; 135 | esac 136 | done 137 | -------------------------------------------------------------------------------- /install/restart_cluster.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | 5 | update_solana_version() { 6 | sudo -i -u solana solana-install init 1.7.12 7 | } 8 | 9 | create_snapshot_form_ledger() { 10 | sudo -i -u solana solana-ledger-tool --ledger /mnt/ledger create-snapshot 95038710 /mnt/ledger/shapshots/ --snapshot-archive-path /mnt/ledger/shapshots/ --hard-fork 95038710 --wal-recovery-mode skip_any_corrupted_record 11 | } 12 | 13 | create_config() { 14 | 15 | echo "### Update packages... ###" 16 | apt update 17 | echo "### Install ansible, curl, unzip... ###" 18 | apt install ansible curl unzip --yes 19 | 20 | ansible-galaxy collection install ansible.posix 21 | ansible-galaxy collection install community.general 22 | 23 | echo "### Download Solana validator manager" 24 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$1.zip" 25 | echo "starting $cmd" 26 | curl -fsSL "$cmd" --output sv_manager.zip 27 | echo "### Unpack Solana validator manager ###" 28 | unzip ./sv_manager.zip -d . 29 | 30 | mv sv-manager* sv_manager 31 | rm ./sv_manager.zip 32 | cd ./sv_manager || exit 33 | cp -r ./inventory_example ./inventory 34 | 35 | echo "### Which cluster do you want to configure? ###" 36 | select cluster in "mainnet-beta" "testnet"; do 37 | case $cluster in 38 | mainnet-beta ) cluster_environment="mainnet-beta"; break;; 39 | testnet ) cluster_environment="testnet"; break;; 40 | esac 41 | done 42 | 43 | 44 | echo "### Please type your validator name: " 45 | read VALIDATOR_NAME 46 | echo "### Please type the full path to your validator keys: " 47 | read PATH_TO_VALIDATOR_KEYS 48 | 49 | if [ ! -f "$PATH_TO_VALIDATOR_KEYS/validator-keypair.json" ] 50 | then 51 | echo "key $PATH_TO_VALIDATOR_KEYS/validator-keypair.json not found. Pleas verify and run the script again" 52 | exit 53 | fi 54 | 55 | read -e -p "### Please tell which user is running validator: " SOLANA_USER 56 | 57 | ansible-playbook --connection=local --inventory ./inventory --limit local playbooks/pb_config.yaml --extra-vars "{'host_hosts': 'local', \ 58 | 'solana_user': '$SOLANA_USER', \ 59 | 'validator_name':'$VALIDATOR_NAME', \ 60 | 'secrets_path': '$PATH_TO_VALIDATOR_KEYS', \ 61 | 'flat_path': 'True', \ 62 | 'cluster_environment':'$cluster_environment'\ 63 | }" 64 | 65 | remove ansible --yes 66 | 67 | } 68 | 69 | update_validator() { 70 | 71 | rm -rf sv_manager/ 72 | 73 | echo "### Update packages... ###" 74 | apt update 75 | echo "### Install ansible, curl, unzip... ###" 76 | apt install ansible curl unzip --yes 77 | 78 | ansible-galaxy collection install ansible.posix 79 | ansible-galaxy collection install community.general 80 | 81 | echo "### Download Solana validator manager" 82 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$1.zip" 83 | echo "starting $cmd" 84 | curl -fsSL "$cmd" --output sv_manager.zip 85 | echo "### Unpack Solana validator manager ###" 86 | unzip ./sv_manager.zip -d . 87 | 88 | mv sv-manager* sv_manager 89 | rm ./sv_manager.zip 90 | cd ./sv_manager || exit 91 | cp -r ./inventory_example ./inventory 92 | 93 | ansible-playbook --connection=local --inventory ./inventory --limit local playbooks/pb_cluster_restart.yaml --extra-vars "@/etc/sv_manager/sv_manager.conf" --extra-vars 'host_hosts=local' 94 | 95 | 96 | apt remove ansible --yes 97 | 98 | systemctl daemon-reload 99 | systemctl restart solana-validator 100 | } 101 | 102 | process() { 103 | update_solana_version 104 | create_snapshot_form_ledger 105 | update_validator "${1:-latest}" "${2:-""}" 106 | } 107 | 108 | if [ -f /etc/sv_manager/sv_manager.conf ] 109 | then 110 | echo "### Validator has been already installed. Start update?" 111 | select yn in "Yes" "No"; do 112 | case $yn in 113 | Yes ) update_validator "${1:-latest}" "${2:-""}"; break;; 114 | No ) echo "### Aborting update. No changes are made on the system."; exit;; 115 | esac 116 | done 117 | else 118 | echo '### Validator is not installed, or the version is too old. ###' 119 | echo '### should we create valiadtor config. ###' 120 | select yn in "Yes" "No"; do 121 | case $yn in 122 | Yes ) create_config "${1:-latest}"; break;; 123 | No ) echo "### Aborting update. No changes are made on the system."; exit;; 124 | esac 125 | done 126 | 127 | fi 128 | -------------------------------------------------------------------------------- /install/update_agave_validator_version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | echo "###################### WARNING!!! ###################################" 5 | echo "### This script will perform the following operations: ###" 6 | echo "### * wait for validator restart window ###" 7 | echo "### * validator binaries ###" 8 | echo "### * restart sys tuner service ###" 9 | echo "### * restart validator service ###" 10 | echo "### * wait for catchup ###" 11 | echo "#####################################################################" 12 | 13 | update_validator() { 14 | sudo -i -u solana agave-install init "$version" 15 | echo "Version "$version" successfully downloaded" 16 | systemctl restart solana-sys-tuner 17 | if [ ! -f /mnt/solana/ledger/admin.rpc ] 18 | then 19 | sudo -i -u solana agave-validator --ledger /mnt/solana/ledger wait-for-restart-window 20 | systemctl restart solana-validator 21 | else 22 | echo "Ledger directory not found. Restart your validator service manually." 23 | fi 24 | 25 | } 26 | 27 | catchup_info() { 28 | 29 | while true; do 30 | 31 | sudo -i -u solana solana catchup --our-localhost 32 | status=$? 33 | 34 | if [ $status -eq 0 ] 35 | then 36 | exit 0 37 | fi 38 | 39 | echo "waiting next 30 seconds for rpc" 40 | sleep 30 41 | 42 | done 43 | 44 | } 45 | 46 | version=${1:-latest} 47 | 48 | echo "updating to version $version" 49 | update_validator 50 | catchup_info 51 | -------------------------------------------------------------------------------- /install/update_monitoring.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | update_monitoring() { 5 | cd 6 | rm -rf sv_manager/ 7 | if [[ $(grep cluster_environment /etc/sv_manager/sv_manager.conf | cut -d':' -f2) == *"testnet"* ]]; 8 | then 9 | inventory=testnet 10 | else 11 | inventory=mainnet 12 | fi 13 | 14 | if [[ $(which apt | wc -l) -gt 0 ]] 15 | then 16 | pkg_manager=apt 17 | elif [[ $(which yum | wc -l) -gt 0 ]] 18 | then 19 | pkg_manager=yum 20 | fi 21 | 22 | echo "### Update packages... ###" 23 | $pkg_manager update 24 | echo "### Install ansible, curl, unzip... ###" 25 | $pkg_manager install ansible curl unzip --yes 26 | 27 | echo "### Download Solana validator manager" 28 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$1.zip" 29 | echo "starting $cmd" 30 | curl -fsSL "$cmd" --output sv_manager.zip 31 | echo "### Unpack Solana validator manager ###" 32 | unzip ./sv_manager.zip -d . 33 | 34 | mv sv-manager* sv_manager 35 | rm ./sv_manager.zip 36 | cd ./sv_manager || exit 37 | cp -r ./inventory_example ./inventory 38 | 39 | ansible-playbook --connection=local --inventory ./inventory/$inventory.yaml --limit localhost playbooks/pb_install_monitoring.yaml --tags telegraf.configure,monitoring.script --extra-vars "@/etc/sv_manager/sv_manager.conf" 40 | } 41 | 42 | if [ -f /etc/sv_manager/sv_manager.conf ] 43 | then 44 | echo "### Monitoring has been already installed. Start update?" 45 | select yn in "Yes" "No"; do 46 | case $yn in 47 | Yes ) update_monitoring "${1:-latest}"; break;; 48 | No ) echo "### Aborting update. No changes are made on the system."; exit;; 49 | esac 50 | done 51 | else 52 | echo '### Monitoring is not installed, or the version is too old. ###' 53 | echo '### Please run full install of the latest version using this command: ###' 54 | echo '### /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/mfactory-lab/sv-manager/latest/install/install_monitoring.sh)" ###' 55 | fi 56 | -------------------------------------------------------------------------------- /install/update_test_validator_version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | echo "###################### WARNING!!! ###################################" 5 | echo "### This script will perform the following operations: ###" 6 | echo "### * wait for validator restart window ###" 7 | echo "### * validator binaries ###" 8 | echo "### * restart sys tuner service ###" 9 | echo "### * restart validator service ###" 10 | echo "### * wait for catchup ###" 11 | echo "#####################################################################" 12 | 13 | update_validator() { 14 | sudo -i -u solana solana-install init "$version" 15 | echo "Version "$version" successfully downloaded" 16 | systemctl restart solana-sys-tuner 17 | if [ ! -f /mnt/solana/ledger/admin.rpc ] 18 | then 19 | sudo -i -u solana solana-validator --ledger /mnt/solana/ledger wait-for-restart-window 20 | systemctl restart solana-validator 21 | else 22 | echo "Ledger directory not found. Restart your validator service manually." 23 | fi 24 | 25 | } 26 | 27 | catchup_info() { 28 | 29 | while true; do 30 | 31 | sudo -i -u solana solana catchup --our-localhost 32 | status=$? 33 | 34 | if [ $status -eq 0 ] 35 | then 36 | exit 0 37 | fi 38 | 39 | echo "waiting next 30 seconds for rpc" 40 | sleep 30 41 | 42 | done 43 | 44 | } 45 | 46 | version=${1:-latest} 47 | 48 | echo "updating to version $version" 49 | update_validator 50 | catchup_info 51 | -------------------------------------------------------------------------------- /install/update_validator_node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | # required tags: 4 | # validator.service.solana 5 | # cli.update 6 | 7 | 8 | wait_for_restart_window() { 9 | if [ -d /mnt/ledger ] 10 | then 11 | sudo -i -u solana bash -c "$(echo 'set -x && cd /mnt && solana-validator wait-for-restart-window')" 12 | else 13 | if [ -d /mnt/ramdisk/solana/ledger/ ] 14 | then 15 | sudo -i -u solana bash -c "$(echo 'set -x && cd /mnt/ramdisk/solana/ && solana-validator wait-for-restart-window')" 16 | else 17 | sudo -i -u solana solana-validator wait-for-restart-window 18 | fi 19 | fi 20 | } 21 | 22 | catchup_info() { 23 | 24 | while true; do 25 | 26 | sudo -i -u solana solana catchup .secrets/validator-keypair.json --our-localhost 27 | status=$? 28 | 29 | if [ $status -eq 0 ] 30 | then 31 | exit 0 32 | fi 33 | 34 | echo "waiting next 30 seconds for rpc" 35 | sleep 30 36 | 37 | done 38 | 39 | } 40 | 41 | update_validator() { 42 | 43 | rm -rf sv_manager/ 44 | 45 | if [[ $(which apt | wc -l) -gt 0 ]] 46 | then 47 | pkg_manager=apt 48 | elif [[ $(which yum | wc -l) -gt 0 ]] 49 | then 50 | pkg_manager=yum 51 | fi 52 | 53 | echo "### Update packages... ###" 54 | $pkg_manager update 55 | echo "### Install ansible, curl, unzip... ###" 56 | $pkg_manager install ansible curl unzip --yes 57 | 58 | ansible-galaxy collection install ansible.posix 59 | ansible-galaxy collection install community.general 60 | 61 | echo "### Download Solana validator manager" 62 | cmd="https://github.com/mfactory-lab/sv-manager/archive/refs/tags/$3.zip" 63 | echo "starting $cmd" 64 | curl -fsSL "$cmd" --output sv_manager.zip 65 | echo "### Unpack Solana validator manager ###" 66 | unzip ./sv_manager.zip -d . 67 | 68 | mv sv-manager* sv_manager 69 | rm ./sv_manager.zip 70 | cd ./sv_manager || exit 71 | cp -r ./inventory_example ./inventory 72 | 73 | if [ "$4" = "wait" ] 74 | then 75 | wait_for_restart_window 76 | fi 77 | 78 | ansible-playbook --connection=local --inventory ./inventory --limit localhost playbooks/pb_update_validator.yaml --tags "$2" --extra-vars "@/etc/sv_manager/sv_manager.conf" --extra-vars "version=$1" 79 | 80 | catchup_info 81 | 82 | echo Do you want to Uninstall ansible? 83 | select yn in "Yes" "No"; do 84 | case $yn in 85 | Yes ) $pkg_manager remove ansible --yes; break;; 86 | No ) echo "### Okay, ansible is still installed on this system. ###"; break;; 87 | esac 88 | done 89 | } 90 | 91 | if [ -f /etc/sv_manager/sv_manager.conf ] 92 | then 93 | echo "### Validator has been already installed. Start update?" 94 | select yn in "Yes" "No"; do 95 | case $yn in 96 | Yes ) update_validator "${1:-""}" "${2:-""}" "${3:-latest}" "${4:-wait}"; sobreak;; 97 | No ) echo "### Aborting update. No changes are made on the system."; exit;; 98 | esac 99 | done 100 | else 101 | echo '### Validator is not installed, or the version is too old. ###' 102 | echo '### Please run full install of the latest version using this command: ###' 103 | echo '### /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/mfactory-lab/sv-manager/latest/install/install_monitoring.sh)" ###' 104 | fi 105 | -------------------------------------------------------------------------------- /inventory_example/group_vars/all.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | #backward compatibility 4 | old_swap_file: "/mnt/swap/swapfile" 5 | old_ramdisk_path: /mnt/ramdisk 6 | 7 | # users: 8 | solana_user: 'solana' 9 | ansible_user: 'root' 10 | 11 | #all: 12 | accounts_path: "{{ ramdisk_path }}/accounts" 13 | use_firewall: True 14 | env_path: "{{ solana_home }}/.local/share/solana/install/active_release/bin" 15 | fail_if_no_validator_keypair: True 16 | flat_path: True 17 | incremental_snapshot_interval_slots: 2500 18 | ledger_path: "{{ mount_base_path }}/ledger" 19 | local_secrets_path: '../.secrets' 20 | log_level: "INFO" 21 | lvm_enabled: False 22 | lvm_log_volume_size: '25G' 23 | lvm_vg: 'vg00' 24 | mount_base_path: '/mnt/solana' 25 | max_delinquent_stake: 5 26 | name: "{{ inventory_hostname }}" 27 | open_solana_ports_start: 8000 28 | open_solana_ports_end: 10000 29 | ramdisk_path: "{{ mount_base_path }}/ramdisk" 30 | ramdisk_size_gb: 200 31 | rpc_node: False 32 | set_validator_info: False 33 | secrets_path: "/home/{{ solana_user }}/.secrets" 34 | snapshots_path: "{{ mount_base_path }}/snapshots" 35 | solana_home: "/home/{{ solana_user }}" 36 | solana_rpc_port: 8899 37 | swap_file_size_gb: 64 38 | upload_validator_keys: True 39 | validator_name: "{{ inventory_hostname }}" 40 | validator_log_path: "{{ mount_base_path }}/log" 41 | 42 | #validator info 43 | validator_description: "Bootstrapped with https://sv-manager.thevalidators.io" 44 | validator_homepage: "" 45 | keybase_username: '' 46 | 47 | #monitoring: 48 | telegraf_database: v_metrics 49 | telegraf_urls: http://influx.thevalidators.io:8086 50 | telegraf_username: v_user 51 | telegraf_password: thepassword 52 | 53 | # Sys Tuning 54 | sysctl_params: 55 | - 'net.core.wmem_default=134217728' 56 | - 'vm.max_map_count=1000000' 57 | - 'net.core.wmem_max=134217728' 58 | - 'net.core.rmem_default=134217728' 59 | - 'net.core.rmem_max=134217728' 60 | - 'fs.nr_open=1000000' 61 | - 'vm.swappiness=5' -------------------------------------------------------------------------------- /inventory_example/group_vars/local.yaml: -------------------------------------------------------------------------------- 1 | become: False 2 | ansible_connection: local 3 | ansible_python_interpreter: "{{ ansible_playbook_python }}" 4 | flat_path: True -------------------------------------------------------------------------------- /inventory_example/group_vars/mainnet_validators.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | set_validator_info: False 3 | cluster_environment: 'mainnet-beta' 4 | cluster_rpc_address: 'https://api.mainnet-beta.solana.com' 5 | solana_validator_service: stopped 6 | swap_file_size_gb: 200 7 | entrypoints: 8 | - entrypoint.mainnet-beta.solana.com:8001 9 | - entrypoint2.mainnet-beta.solana.com:8001 10 | - entrypoint3.mainnet-beta.solana.com:8001 11 | - entrypoint4.mainnet-beta.solana.com:8001 12 | - entrypoint5.mainnet-beta.solana.com:8001 13 | known_validators: 14 | - 7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2 15 | - GdnSyH3YtwcxFvQrVVJMm1JhTS4QVX7MFsX56uJLUfiZ 16 | - DE1bawNcRJB9rVm3buyMVfr8mBEoyyu73NBovf2oXJsJ 17 | - CakcnaRDHka2gXyfbEd2d3xsvkJkqsLw2akB3zsN1D2S 18 | solana_metrics_url: 'https://metrics.solana.com:8086,db=mainnet-beta,u=mainnet-beta_write,p=password' 19 | expected_genesis_hash: '5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d' 20 | limit_ledger_size: 50000000 21 | maximum_local_snapshot_age: 2000 22 | solana_version: 1.18.18 23 | jito_version: 1.18.15 24 | agave_version: 2.0.2 25 | extra_params: 26 | -------------------------------------------------------------------------------- /inventory_example/group_vars/remote.yaml: -------------------------------------------------------------------------------- 1 | flat_path: False -------------------------------------------------------------------------------- /inventory_example/group_vars/rpc.yaml: -------------------------------------------------------------------------------- 1 | rpc_node: True 2 | ts_service_name: '' 3 | ts_password: '' 4 | ts_host: '' 5 | -------------------------------------------------------------------------------- /inventory_example/group_vars/testnet_validators.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | set_validator_info: False 3 | cluster_environment: 'testnet' 4 | cluster_rpc_address: 'https://api.testnet.solana.com' 5 | solana_validator_service: started 6 | entrypoints: 7 | - entrypoint.testnet.solana.com:8001 8 | - entrypoint2.testnet.solana.com:8001 9 | - entrypoint3.testnet.solana.com:8001 10 | known_validators: 11 | - 5D1fNXzvv5NjV1ysLjirC4WY92RNsVH18vjmcszZd8on 12 | - dDzy5SR3AXdYWVqbDEkVFdvSPCtS9ihF5kJkHCtXoFs 13 | - Ft5fbkqNa76vnsjYNwjDZUXoTWpP7VYm3mtsaQckQADN 14 | - eoKpUABi59aT4rR9HGS3LcMecfut9x7zJyodWWP43YQ 15 | - 9QxCLckBiJc783jnMvXZubK4wH86Eqqvashtrwvcsgkv 16 | solana_metrics_url: 'https://metrics.solana.com:8086,db=tds,u=testnet_write,p=c4fa841aa918bf8274e3e2a44d77568d9861b3ea' 17 | expected_genesis_hash: '4uhcVJyU9pJkvQyS88uRDiswHXSCkY3zQawwpjk2NsNY' 18 | limit_ledger_size: 50000000 19 | maximum_local_snapshot_age: 1000 20 | solana_version: 1.18.17 21 | jito_version: 1.18.17 22 | agave_version: 2.0.3 23 | extra_params: 24 | -------------------------------------------------------------------------------- /inventory_example/mainnet.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | all: 3 | children: 4 | mainnet_validators: 5 | children: 6 | remote: 7 | hosts: 8 | server1: 9 | ansible_host: 10 | #any var from group vars: 11 | ramdisk_size_gb: 12 | validator_description: 'Bootstrapped with https://sv-manager.thevalidators.io' 13 | validator_homepage: '' 14 | validator_keybase: '' 15 | server2: 16 | ansible_host: 17 | #any var from group vars: 18 | ramdisk_size_gb: 19 | local: 20 | hosts: 21 | localhost: 22 | #any var from group vars: 23 | validator_name: "" 24 | localhost_rpc: 25 | ansible_host: localhost 26 | force: true 27 | rpc_node: true 28 | solana_validator_service: started -------------------------------------------------------------------------------- /inventory_example/testnet.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | all: 3 | children: 4 | testnet_validators: 5 | children: 6 | remote: 7 | hosts: 8 | server1: 9 | ansible_host: 10 | #any var from group vars: 11 | validator_description: "Bootstrapped with https://sv-manager.thevalidators.io" 12 | validator_homepage: "" 13 | validator_keybase: "" 14 | server2: 15 | ansible_host: 16 | local: 17 | hosts: 18 | localhost: 19 | #any var from group vars: 20 | validator_description: "Bootstrapped with https://sv-manager.thevalidators.io" 21 | validator_homepage: "" 22 | validator_keybase: "" 23 | localhost_rpc: 24 | ansible_host: localhost 25 | force: true 26 | rpc_node: true 27 | solana_validator_service: started -------------------------------------------------------------------------------- /playbooks/pb_cluster_restart.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: restart cluster 4 | hosts: "{{ host_hosts | default('remote') }}" 5 | become: yes 6 | roles: 7 | - restart_cluster 8 | -------------------------------------------------------------------------------- /playbooks/pb_config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install or update config file 3 | hosts: all 4 | become: yes 5 | roles: 6 | - solana_config 7 | -------------------------------------------------------------------------------- /playbooks/pb_debug.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install solana validator 3 | hosts: all 4 | become: yes 5 | 6 | # vars_prompt: 7 | # - name: validator_key_path 8 | # prompt: Where are your keys? 9 | # private: no 10 | # - name: validator_name 11 | # prompt: What is your validator name? 12 | # private: no 13 | # - name: lvm_enabled 14 | # prompt: LVM enabled (Yes/No)? 15 | # default: No 16 | # private: no 17 | 18 | tasks: 19 | # - name: debug variables 20 | # debug: 21 | # var: "{{ item }}" 22 | # with_items: 23 | # - secrets_path 24 | # - ansible_user 25 | # - accounts_path 26 | # - cluster_environment 27 | # - name 28 | # - entrypoint 29 | # - ledger_path 30 | - name: show name 31 | debug: 32 | msg: "{{ hostvars[inventory_hostname].cluster_environment }}" 33 | - name: show cpu 34 | debug: 35 | msg: "{{ group_names }}" -------------------------------------------------------------------------------- /playbooks/pb_install_agave_validator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install solana validator 3 | hosts: all 4 | become: yes 5 | roles: 6 | - check_node 7 | - configure_ubuntu 8 | - agave_cli 9 | - solana_validator_bootstrap 10 | - monitoring 11 | vars: 12 | - agave: yes 13 | -------------------------------------------------------------------------------- /playbooks/pb_install_jito_validator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install solana validator 3 | hosts: all 4 | become: yes 5 | roles: 6 | - check_node 7 | - configure_ubuntu 8 | - solana_cli 9 | - solana_validator_bootstrap 10 | - monitoring 11 | vars: 12 | - jito: yes 13 | -------------------------------------------------------------------------------- /playbooks/pb_install_monitoring.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install monitoring 3 | hosts: all 4 | become: yes 5 | roles: 6 | - monitoring 7 | -------------------------------------------------------------------------------- /playbooks/pb_install_validator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install solana validator 3 | hosts: all 4 | become: yes 5 | roles: 6 | - check_node 7 | - configure_ubuntu 8 | - solana_cli 9 | - solana_validator_bootstrap 10 | - monitoring 11 | -------------------------------------------------------------------------------- /playbooks/pb_stop_validator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: stop solana validator 4 | hosts: "{{ host_hosts | default('remote') }}" 5 | become: yes 6 | tags: 7 | - validator.cluster.restart 8 | roles: 9 | - solana_validator_restart 10 | -------------------------------------------------------------------------------- /playbooks/pb_update_validator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: update solana validator 3 | hosts: all 4 | serial: 1 5 | become: yes 6 | roles: 7 | - solana_cli 8 | - solana_validator_bootstrap 9 | - solana_validator_restart 10 | -------------------------------------------------------------------------------- /roles/agave_cli/tasks/install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: create download dir 3 | file: 4 | path: /tmp/solana 5 | state: directory 6 | owner: "{{ solana_user }}" 7 | group: "{{ solana_user }}" 8 | tags: 9 | - cli.install 10 | 11 | - name: install solana 12 | block: 13 | - name: download latest solana release installer 14 | get_url: 15 | url: "https://release.anza.xyz/v{{ agave_version | default('stable') }}/install" 16 | dest: /tmp/solana/ 17 | mode: 0755 18 | 19 | - name: run solana installer 20 | shell: /tmp/solana/install 21 | become: yes 22 | become_user: "{{ solana_user }}" 23 | tags: 24 | - cli.install 25 | 26 | - name: remove installer 27 | file: 28 | path: /tmp/solana 29 | state: absent 30 | tags: 31 | - cli.install 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /roles/agave_cli/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: set force install fact 3 | set_fact: 4 | force: "{{ force | default('false') }}" 5 | tags: 6 | - cli 7 | - cli.install 8 | - cli.update 9 | 10 | - name: check solana cli installed 11 | stat: 12 | path: /home/solana/.local/share/solana/install/active_release/bin/agave-install 13 | register: solana_exists 14 | tags: 15 | - cli 16 | - cli.install 17 | - cli.update 18 | 19 | - name: install solana cli 20 | import_tasks: install.yaml 21 | tags: 22 | - cli 23 | - cli.install 24 | when: force == 'true' or not solana_exists.stat.exists 25 | 26 | - name: update solana cli 27 | import_tasks: update.yaml 28 | tags: 29 | - cli 30 | - cli.update 31 | when: force != 'true' and solana_exists.stat.exists 32 | -------------------------------------------------------------------------------- /roles/agave_cli/tasks/update.yaml: -------------------------------------------------------------------------------- 1 | - name: DEBUG 2 | debug: 3 | msg: "Updating Solana to {{ agave_version }}" 4 | 5 | - name: update solana (agave) 6 | shell: "agave-install init {{ agave_version }}" 7 | become: yes 8 | become_user: "{{ solana_user }}" 9 | environment: 10 | PATH: "{{ solana_home }}/.local/share/solana/install/active_release/bin" 11 | tags: 12 | - cli.update 13 | -------------------------------------------------------------------------------- /roles/check_node/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Build flat keys path 4 | set_fact: 5 | secrets_local_path: "{{ local_secrets_path }}" 6 | when: flat_path|bool 7 | tags: 8 | - check.keys 9 | 10 | - name: Build non flat keys path 11 | set_fact: 12 | secrets_local_path: "{{ local_secrets_path }}/{{ validator_name }}/solana" 13 | when: not flat_path|bool 14 | tags: 15 | - check.keys 16 | 17 | - name: Check validator-keypair exists locally 18 | stat: 19 | path: "{{ secrets_local_path }}/validator-keypair.json" 20 | connection: local 21 | become: no 22 | delegate_to: localhost 23 | register: validator_keypair_exists 24 | tags: 25 | - check.keys 26 | 27 | - name: Fail if no validator-keypair exists locally 28 | fail: 29 | msg: "No validator-keypair exists locally {{ secrets_local_path }}" 30 | when: not validator_keypair_exists.stat.exists 31 | and not rpc_node 32 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/ansible_user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create ansible user 3 | user: 4 | name: ansible 5 | state: present 6 | create_home: yes 7 | shell: /bin/bash 8 | skeleton: /etc/skel 9 | tags: 10 | - config.ansible_user 11 | 12 | - name: Allow 'ansible' group to have passwordless sudo 13 | lineinfile: 14 | path: /etc/sudoers 15 | state: present 16 | regexp: '^%ansible' 17 | line: '%ansible ALL=(ALL) NOPASSWD: ALL' 18 | validate: 'visudo -cf %s' 19 | tags: 20 | - config.ansible_user 21 | 22 | - name: Create .ssh folder 23 | file: 24 | path: /home/ansible/.ssh 25 | state: directory 26 | owner: ansible 27 | group: ansible 28 | tags: 29 | - config.ansible_user 30 | 31 | - name: Put the public key of ansible-master 32 | lineinfile: 33 | path: /home/ansible/.ssh/authorized_keys 34 | line: "{{ lookup('file', '~/.ssh/id_rsa.pub') }}" 35 | create: yes 36 | tags: 37 | - config.ansible_user 38 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/cpu_governor.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: check is real cpu 3 | stat: 4 | path: /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 5 | register: scaling_governor_exists 6 | tags: 7 | - config.cpu 8 | 9 | - name: set cpu governor to performance 10 | shell: echo "performance" | sudo tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor 11 | become: yes 12 | become_user: root 13 | tags: 14 | - config.cpu 15 | when: scaling_governor_exists.stat.exists 16 | 17 | # - name: Populate service facts 18 | # ansible.builtin.service_facts: 19 | # tags: 20 | # - config.cpu 21 | 22 | # - name: Disable ondemand service 23 | # systemd: 24 | # name: ondemand 25 | # state: stopped 26 | # enabled: no 27 | # tags: 28 | # - config.cpu 29 | # when: "'ondemand' in services" 30 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/fail2ban.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install fail2ban package 3 | package: 4 | name: 5 | - fail2ban 6 | state: present 7 | tags: 8 | - config.fail2ban 9 | 10 | - name: Create jail.local 11 | template: 12 | src: jail.local.j2 13 | dest: /etc/fail2ban/jail.local 14 | mode: 0644 15 | owner: root 16 | group: root 17 | tags: 18 | - config.fail2ban 19 | 20 | - name: Restart fail2ban 21 | systemd: 22 | name: fail2ban 23 | state: restarted 24 | tags: 25 | - config.fail2ban -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/firewall.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install firewall package 3 | package: 4 | name: 5 | - ufw 6 | state: present 7 | tags: 8 | - config.firewall 9 | 10 | - name: Deny all ingress connections 11 | ufw: 12 | policy: deny 13 | direction: incoming 14 | tags: 15 | - config.firewall 16 | 17 | - name: allow ingress ssh 18 | ufw: 19 | rule: allow 20 | port: ssh 21 | proto: tcp 22 | direction: in 23 | tags: 24 | - config.firewall 25 | 26 | - name: allow rpc ingress port 27 | ufw: 28 | rule: allow 29 | proto: tcp 30 | direction: in 31 | port: '{{ solana_rpc_port }}' 32 | tags: 33 | - config.firewall 34 | 35 | - name: allow ingress solana udp ports 36 | ufw: 37 | rule: allow 38 | proto: udp 39 | direction: in 40 | port: '{{ open_solana_ports_start }}:{{ open_solana_ports_end }}' 41 | tags: 42 | - config.firewall 43 | 44 | - name: allow ingress solana tcp ports 45 | ufw: 46 | rule: allow 47 | proto: tcp 48 | direction: in 49 | port: '{{ open_solana_ports_start }}:{{ open_solana_ports_end }}' 50 | tags: 51 | - config.firewall 52 | 53 | - name: deny out from any to 10.0.0.0/8 54 | ufw: 55 | rule: deny 56 | direction: out 57 | src: '{{ item }}' 58 | loop: 59 | - 10.0.0.0/8 60 | - 172.16.0.0/12 61 | - 192.168.0.0/16 62 | - 100.64.0.0/10 63 | - 198.18.0.0/15 64 | - 169.254.0.0/16 65 | tags: 66 | - config.firewall 67 | 68 | - name: Enable ufw 69 | ufw: 70 | state: enabled 71 | tags: 72 | - config.firewall 73 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/lvm.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: check lvm configured 3 | stat: 4 | path: "/.lvm.done" 5 | tags: 6 | - config.lvm 7 | register: lvm_configured 8 | 9 | - name: create mount base path 10 | file: 11 | path: "{{ mount_base_path }}" 12 | state: directory 13 | mode: '0755' 14 | tags: 15 | - config.lvm 16 | 17 | - name: create ledger mount point 18 | file: 19 | path: "{{ ledger_path }}" 20 | state: directory 21 | mode: '0755' 22 | tags: 23 | - config.lvm 24 | 25 | - name: create ledger log point 26 | file: 27 | path: "{{ validator_log_path }}" 28 | state: directory 29 | mode: '0755' 30 | tags: 31 | - config.lvm 32 | 33 | - name: configured lvm 34 | block: 35 | - name: Add solana lvm log volume 36 | lvol: 37 | vg: "{{ lvm_vg }}" 38 | lv: log 39 | size: "{{ lvm_log_volume_size }}" 40 | state: present 41 | 42 | - name: Create a ext4 filesystem on log dir 43 | filesystem: 44 | fstype: ext4 45 | dev: "/dev/{{ lvm_vg }}/log" 46 | 47 | - name: Create log dir mount point 48 | file: 49 | path: "{{ mount_base_path }}/log" 50 | state: directory 51 | mode: '0755' 52 | 53 | - name: Write log dir entry in fstab 54 | ansible.posix.mount: 55 | path: "{{ mount_base_path }}/log" 56 | src: "/dev/{{ lvm_vg }}/log" 57 | fstype: ext4 58 | state: mounted 59 | 60 | - name: Add solana lvm log volume 61 | lvol: 62 | vg: "{{ lvm_vg }}" 63 | lv: log 64 | size: "{{ lvm_log_volume_size }}" 65 | state: present 66 | 67 | - name: Create a ext4 filesystem on log dir 68 | filesystem: 69 | fstype: ext4 70 | dev: "/dev/{{ lvm_vg }}/log" 71 | 72 | - name: Create log dir mount point 73 | file: 74 | path: "{{ mount_base_path }}/log" 75 | state: directory 76 | mode: '0755' 77 | 78 | - name: Write log dir entry in fstab 79 | ansible.posix.mount: 80 | path: "{{ mount_base_path }}/log" 81 | src: "/dev/{{ lvm_vg }}/log" 82 | fstype: ext4 83 | state: mounted 84 | 85 | - name: Add solana lvm swap volume 86 | lvol: 87 | vg: "{{ lvm_vg }}" 88 | lv: swap 89 | size: "{{ (swap_file_size_gb | int + 5) | int }}G" 90 | state: present 91 | 92 | - name: Create a ext4 filesystem on swap dir 93 | filesystem: 94 | fstype: ext4 95 | dev: "/dev/{{ lvm_vg }}/swap" 96 | 97 | - name: Create swap dir mount point 98 | file: 99 | path: "{{ mount_base_path }}/swap" 100 | state: directory 101 | mode: '0755' 102 | 103 | - name: Write swap dir entry in fstab 104 | ansible.posix.mount: 105 | path: "{{ mount_base_path }}/swap" 106 | src: "/dev/{{ lvm_vg }}/swap" 107 | fstype: ext4 108 | state: mounted 109 | 110 | - name: Add solana lvm ledger volume 111 | lvol: 112 | vg: "{{ lvm_vg }}" 113 | lv: ledger 114 | size: 100%FREE 115 | state: present 116 | 117 | - name: Create a ext4 filesystem on ledger 118 | filesystem: 119 | fstype: ext4 120 | dev: "/dev/{{ lvm_vg }}/ledger" 121 | 122 | - name: Write ledger entry in fstab 123 | ansible.posix.mount: 124 | path: "{{ ledger_path }}" 125 | src: "/dev/{{ lvm_vg }}/ledger" 126 | fstype: ext4 127 | state: mounted 128 | 129 | - name: set lvm configured 130 | file: 131 | path: "/.lvm.done" 132 | state: touch 133 | 134 | when: not lvm_configured.stat.exists 135 | tags: 136 | - config.lvm 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Configure lvm 3 | import_tasks: lvm.yaml 4 | when: lvm_enabled|bool 5 | tags: 6 | config.lvm 7 | 8 | - name: Install additional packages 9 | import_tasks: packages.yaml 10 | tags: 11 | - config.packages 12 | 13 | - name: Create swap 14 | import_tasks: swap.yaml 15 | tags: 16 | - config.swap 17 | when: swap_file_size_gb | int > 0 18 | 19 | - name: Create ramdisk 20 | import_tasks: ramdisk.yaml 21 | tags: 22 | - config.ramdisk 23 | when: ramdisk_size_gb | int > 0 24 | 25 | - name: Config cpu governor 26 | import_tasks: cpu_governor.yaml 27 | tags: 28 | - config.cpu 29 | 30 | - name: Config firewall 31 | import_tasks: firewall.yaml 32 | tags: 33 | - config.firewall 34 | when: 35 | use_firewall | bool 36 | 37 | - name: Config file2ban 38 | import_tasks: fail2ban.yaml 39 | tags: 40 | - config.fail2ban 41 | 42 | - name: Create ansible user 43 | import_tasks: ansible_user.yaml 44 | tags: 45 | - config.ansible_user 46 | when: 47 | - "'local' not in group_names" 48 | 49 | - name: Create solana user 50 | import_tasks: solana_user.yaml 51 | tags: 52 | - config.solana_user 53 | - validator.service.solana 54 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/packages.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: packages | ensure apt list dir exists 4 | file: 5 | path: /var/lib/apt/lists/ 6 | state: directory 7 | mode: 0755 8 | 9 | - name: Update apt cache 10 | become: yes 11 | apt: 12 | update_cache: yes 13 | 14 | - name: Install additional packages 15 | become: yes 16 | apt: 17 | pkg: 18 | - gpg 19 | - gpg-agent 20 | - python3 21 | - python3-pip 22 | - python3-venv 23 | - ufw 24 | - rsyslog 25 | tags: 26 | - config.packages 27 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/ramdisk.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: set force install fact 3 | set_fact: 4 | force: "{{ force | default('false') }}" 5 | 6 | - name: check ramdisk mount point extsts 7 | stat: 8 | path: "{{ ramdisk_path }}" 9 | register: ramdisk_exists 10 | tags: 11 | - config.ramdisk 12 | 13 | - name: check OLD ramdisk mount point extsts 14 | stat: 15 | path: "{{ old_ramdisk_path }}" 16 | register: old_ramdisk_exists 17 | tags: 18 | - config.ramdisk 19 | 20 | - name: create ramdisk mount point 21 | file: 22 | path: "{{ ramdisk_path }}" 23 | state: directory 24 | mode: '0755' 25 | tags: 26 | - config.ramdisk.directory 27 | when: force or (not ramdisk_exists.stat.exists and not old_ramdisk_exists.stat.exists) 28 | 29 | - name: Write ramdisk entry in fstab 30 | mount: name={{ ramdisk_path }} 31 | src=tmpfs 32 | fstype=tmpfs 33 | opts=nodev,nosuid,noexec,nodiratime,size={{ ramdisk_size_gb }}G 34 | passno=0 35 | dump=0 36 | state=mounted 37 | tags: 38 | - config.ramdisk.fstab 39 | when: force or (not ramdisk_exists.stat.exists and not old_ramdisk_exists.stat.exists) 40 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/solana_user.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Ensure group solana exists 3 | ansible.builtin.group: 4 | name: "{{ solana_user }}" 5 | state: present 6 | 7 | - name: Create user solana 8 | user: 9 | name: "{{ solana_user }}" 10 | state: present 11 | create_home: yes 12 | shell: /bin/bash 13 | skeleton: /etc/skel 14 | force: yes 15 | groups: 16 | - "{{ solana_user }}" 17 | - syslog 18 | tags: 19 | - config.solana_user 20 | - validator.service.solana 21 | 22 | - name: Create secrets dir 23 | become: yes 24 | become_user: "{{ solana_user }}" 25 | file: 26 | path: "{{ secrets_path }}" 27 | state: directory 28 | mode: '0700' 29 | tags: 30 | - config.solana_user 31 | - validator.service.solana 32 | 33 | - name: Create ledger directory 34 | become: yes 35 | file: 36 | path: "{{ ledger_path }}" 37 | owner: "{{ solana_user }}" 38 | group: "{{ solana_user }}" 39 | state: directory 40 | mode: '0755' 41 | tags: 42 | - config.solana_user 43 | - validator.service.solana 44 | 45 | - name: Create accounts directory 46 | become: yes 47 | file: 48 | path: "{{ accounts_path }}" 49 | owner: "{{ solana_user }}" 50 | group: "{{ solana_user }}" 51 | state: directory 52 | mode: '0755' 53 | tags: 54 | - config.solana_user 55 | - validator.service.solana 56 | 57 | - name: Create snapshots directory 58 | become: yes 59 | file: 60 | path: "{{ snapshots_path }}" 61 | owner: "{{ solana_user }}" 62 | group: "{{ solana_user }}" 63 | state: directory 64 | mode: '0755' 65 | tags: 66 | - config.solana_user 67 | - validator.service.solana 68 | 69 | - name: Create logs directory 70 | become: yes 71 | file: 72 | path: "{{ validator_log_path }}" 73 | owner: "{{ solana_user }}" 74 | group: "{{ solana_user }}" 75 | state: directory 76 | mode: '0755' 77 | tags: 78 | - config.solana_user 79 | - validator.service.solana 80 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/tasks/swap.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: set force install fact 3 | set_fact: 4 | force: "{{ force | default('false') }}" 5 | tags: 6 | - config.swap 7 | 8 | - name: check swap file exists 9 | stat: 10 | path: "{{ mount_base_path }}/swap/swapfile" 11 | tags: 12 | - config.swap 13 | register: swap_file_exists 14 | 15 | - name: check OLD swap file exists 16 | stat: 17 | path: "{{ old_swap_file }}" 18 | tags: 19 | - config.swap 20 | register: old_swap_file_exists 21 | 22 | - name: Create swap dir mount point 23 | file: 24 | path: "{{ mount_base_path }}/swap" 25 | state: directory 26 | mode: '0755' 27 | when: force or (not swap_file_exists.stat.exists and not old_swap_file_exists.stat.exists) 28 | 29 | - name: disable current swap 30 | command: swapoff -a 31 | tags: 32 | - config.swap.file.disable 33 | when: force or (not swap_file_exists.stat.exists and not old_swap_file_exists.stat.exists) 34 | 35 | - name: Create swap file 36 | command: fallocate -l {{ swap_file_size_gb }}G {{ mount_base_path }}/swap/swapfile 37 | tags: 38 | - config.swap.file.create 39 | when: force or (not swap_file_exists.stat.exists and not old_swap_file_exists.stat.exists) 40 | 41 | - name: Change swap file permissions 42 | file: path="{{ mount_base_path }}/swap/swapfile" 43 | owner=root 44 | group=root 45 | mode=0600 46 | tags: 47 | - config.swap.file.permissions 48 | when: force or (not swap_file_exists.stat.exists and not old_swap_file_exists.stat.exists) 49 | 50 | - name: Make swap file 51 | command: "mkswap {{ mount_base_path }}/swap/swapfile" 52 | tags: 53 | - config.swap.file.mkswap 54 | when: force or (not swap_file_exists.stat.exists and not old_swap_file_exists.stat.exists) 55 | 56 | - name: Write swap entry in fstab 57 | mount: name=none 58 | src={{ mount_base_path }}/swap/swapfile 59 | fstype=swap 60 | opts=sw 61 | passno=0 62 | dump=0 63 | state=present 64 | tags: 65 | - config.swap.fstab 66 | when: force or (not swap_file_exists.stat.exists and not old_swap_file_exists.stat.exists) 67 | 68 | - name: Mount swap 69 | command: "swapon {{ mount_base_path }}/swap/swapfile" 70 | tags: 71 | - config.swap.file.swapon 72 | when: force or (not swap_file_exists.stat.exists and not old_swap_file_exists.stat.exists) 73 | -------------------------------------------------------------------------------- /roles/configure_ubuntu/templates/jail.local.j2: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | bantime.increment = true 3 | bantime = 30m 4 | ignoreip = 127.0.0.1/8 -------------------------------------------------------------------------------- /roles/monitoring/files/cluster_monitoring_library.py: -------------------------------------------------------------------------------- 1 | import solana_rpc as rpc 2 | 3 | 4 | def get_apr_from_rewards(rewards_data): 5 | result = [] 6 | 7 | if rewards_data is not None: 8 | if 'epochRewards' in rewards_data: 9 | epoch_rewards = rewards_data['epochRewards'] 10 | for reward in epoch_rewards: 11 | result.append({ 12 | 'percent_change': reward['percentChange'], 13 | 'apr': reward['apr'] 14 | }) 15 | 16 | return result 17 | 18 | 19 | def calc_single_apy(apr, percent_change): 20 | epoch_count = apr / percent_change 21 | result = ((1 + percent_change / 100) ** epoch_count - 1) * 100 22 | return result 23 | 24 | 25 | def calc_apy_list_from_apr(apr_per_epoch): 26 | l_apy = [] 27 | 28 | for item in apr_per_epoch: 29 | apy = calc_single_apy(item['apr'], item['percent_change']) 30 | l_apy.append(apy) 31 | 32 | return l_apy 33 | 34 | 35 | def process(validators): 36 | 37 | data = [] 38 | 39 | for validator in validators: 40 | rewards_data = rpc.load_stake_account_rewards(validator['stake_account']) 41 | apr_per_epoch = get_apr_from_rewards(rewards_data) 42 | apy_per_epoch = calc_apy_list_from_apr(apr_per_epoch) 43 | data.append(apy_per_epoch) 44 | 45 | return data 46 | -------------------------------------------------------------------------------- /roles/monitoring/files/common.py: -------------------------------------------------------------------------------- 1 | from pprint import pprint 2 | import json 3 | import numpy 4 | import time 5 | 6 | 7 | class ValidatorConfig: 8 | def __init__(self, 9 | validator_name: str, 10 | secrets_path: str, 11 | local_rpc_address: str, 12 | remote_rpc_address: str, 13 | cluster_environment: str, 14 | debug_mode: bool): 15 | self.validator_name = validator_name 16 | self.secrets_path = secrets_path 17 | self.local_rpc_address = local_rpc_address 18 | self.remote_rpc_address = remote_rpc_address 19 | self.cluster_environment = cluster_environment 20 | self.debug_mode = debug_mode 21 | 22 | 23 | class JsonEncoder(json.JSONEncoder): 24 | """ Special json encoder for numpy types """ 25 | def default(self, obj): 26 | if isinstance(obj, (numpy.int_, numpy.intc, numpy.intp, numpy.int8, 27 | numpy.int16, numpy.int32, numpy.int64, numpy.uint8, 28 | numpy.uint16, numpy.uint32, numpy.uint64)): 29 | return int(obj) 30 | elif isinstance(obj, (numpy.float_, numpy.float16, numpy.float32, 31 | numpy.float64)): 32 | return float(obj) 33 | elif isinstance(obj, (numpy.ndarray,)): 34 | return obj.tolist() 35 | return json.JSONEncoder.default(self, obj) 36 | 37 | 38 | def debug(config: ValidatorConfig, data): 39 | if config.debug_mode: 40 | pprint(data) 41 | 42 | 43 | def print_json(data): 44 | print(json.dumps(data, cls=JsonEncoder)) 45 | 46 | 47 | def measurement_from_fields(name, data, tags, config, legacy_tags=None): 48 | if legacy_tags is None: 49 | legacy_tags = {} 50 | data.update({"cluster_environment": config.cluster_environment}) 51 | measurement = { 52 | "measurement": name, 53 | "time": round(time.time() * 1000), 54 | "monitoring_version": "3.2.0", 55 | "cluster_environment": config.cluster_environment, 56 | "fields": data, 57 | "tags": tags 58 | } 59 | 60 | measurement.update(legacy_tags) 61 | 62 | return measurement 63 | -------------------------------------------------------------------------------- /roles/monitoring/files/measurement_tds_info.py: -------------------------------------------------------------------------------- 1 | import time 2 | import solana_rpc as rpc 3 | from common import debug 4 | from common import ValidatorConfig 5 | import statistics 6 | import numpy as np 7 | import tds_info as tds 8 | from common import measurement_from_fields 9 | 10 | def load_data(config: ValidatorConfig): 11 | identity_account_pubkey = rpc.load_identity_account_pubkey(config) 12 | default = [] 13 | tds_data = default 14 | tds_data = tds.load_tds_info(config, identity_account_pubkey) 15 | 16 | result = { 17 | 'identity_account_pubkey': identity_account_pubkey, 18 | 'tds_data': tds_data 19 | } 20 | 21 | debug(config, str(result)) 22 | 23 | return result 24 | 25 | def calculate_influx_fields(data): 26 | if data is None: 27 | result = {"tds_info": 0} 28 | else: 29 | identity_account_pubkey = data['identity_account_pubkey'] 30 | 31 | result = data['tds_data'] 32 | return result 33 | 34 | def calculate_output_data(config: ValidatorConfig): 35 | data = load_data(config) 36 | 37 | tags = { 38 | "validator_identity_pubkey": data['identity_account_pubkey'], 39 | "validator_name": config.validator_name, 40 | "cluster_environment": config.cluster_environment 41 | } 42 | 43 | 44 | measurement = measurement_from_fields( 45 | "tds_info", 46 | calculate_influx_fields(data), 47 | tags, 48 | config 49 | ) 50 | return measurement 51 | -------------------------------------------------------------------------------- /roles/monitoring/files/measurement_validator_info.py: -------------------------------------------------------------------------------- 1 | import time 2 | import solana_rpc as rpc 3 | from common import debug 4 | from common import ValidatorConfig 5 | import statistics 6 | import numpy as np 7 | from common import measurement_from_fields 8 | 9 | 10 | def get_metrics_from_vote_account_item(item): 11 | return { 12 | 'epoch_number': item['epochCredits'][-1][0], 13 | 'credits_epoch': item['epochCredits'][-1][1], 14 | 'credits_previous_epoch': item['epochCredits'][-1][2], 15 | 'activated_stake': item['activatedStake'], 16 | 'credits_epoch_delta': item['epochCredits'][-1][1] - item['epochCredits'][-1][2], 17 | 'commission': item['commission'] 18 | } 19 | 20 | 21 | def find_item_in_vote_accounts_section(identity_account_pubkey, section_parent, section_name): 22 | if section_name in section_parent: 23 | section = section_parent[section_name] 24 | for item in section: 25 | if item['nodePubkey'] == identity_account_pubkey: 26 | return get_metrics_from_vote_account_item(item) 27 | 28 | return None 29 | 30 | 31 | def get_vote_account_metrics(vote_accounts_data, identity_account_pubkey): 32 | """ 33 | get vote metrics from vote account 34 | :return: 35 | voting_status: 0 if validator not found in voting accounts 36 | voting_status: 1 if validator is current 37 | voting_status: 2 if validator is delinquent 38 | 39 | """ 40 | result = find_item_in_vote_accounts_section(identity_account_pubkey, vote_accounts_data, 'current') 41 | if result is not None: 42 | result.update({'voting_status': 1}) 43 | else: 44 | result = find_item_in_vote_accounts_section(identity_account_pubkey, vote_accounts_data, 'delinquent') 45 | if result is not None: 46 | result.update({'voting_status': 2}) 47 | else: 48 | result = {'voting_status': 0} 49 | return result 50 | 51 | 52 | def get_leader_schedule_metrics(leader_schedule_data, identity_account_pubkey): 53 | """ 54 | get metrics about leader slots 55 | """ 56 | if identity_account_pubkey in leader_schedule_data: 57 | return {"leader_slots_this_epoch": len(leader_schedule_data[identity_account_pubkey])} 58 | else: 59 | return {"leader_slots_this_epoch": 0} 60 | 61 | 62 | def get_block_production_metrics(block_production_data, identity_account_pubkey): 63 | try: 64 | item = block_production_data['value']['byIdentity'][identity_account_pubkey] 65 | return { 66 | "slots_done": item[0], 67 | "slots_skipped": item[0] - item[1], 68 | "blocks_produced": item[1] 69 | 70 | } 71 | except: 72 | return {"slots_done": 0, "slots_skipped": 0, "blocks_produced": 0} 73 | 74 | 75 | def get_block_production_cli_metrics(block_production_data_cli, identity_account_pubkey: str): 76 | if 'leaders' in block_production_data_cli: 77 | leaders = block_production_data_cli['leaders'] 78 | skip_rate = [] 79 | my_skip_rate = 0 80 | for leader in leaders: 81 | leader_slots = leader.get('leaderSlots', 0) 82 | if leader_slots > 0: 83 | current_skip_rate = leader.get('skippedSlots', 0) / leader_slots 84 | skip_rate.append(current_skip_rate) 85 | if leader['identityPubkey'] == identity_account_pubkey: 86 | my_skip_rate = current_skip_rate 87 | 88 | result = { 89 | 'leader_skip_rate': my_skip_rate, 90 | 'cluster_min_leader_skip_rate': min(skip_rate), 91 | 'cluster_max_leader_skip_rate': max(skip_rate), 92 | 'cluster_mean_leader_skip_rate': statistics.mean(skip_rate), 93 | 'cluster_median_leader_skip_rate': statistics.median(skip_rate), 94 | } 95 | else: 96 | result = {} 97 | 98 | return result 99 | 100 | 101 | def get_performance_metrics(performance_sample_data, epoch_info_data, leader_schedule_by_identity): 102 | if len(performance_sample_data) > 0: 103 | sample = performance_sample_data[0] 104 | if sample['numSlots'] > 0: 105 | mid_slot_time = sample['samplePeriodSecs'] / sample['numSlots'] 106 | else: 107 | mid_slot_time = 0 108 | current_slot_index = epoch_info_data['slotIndex'] 109 | remaining_time = (epoch_info_data["slotsInEpoch"] - current_slot_index) * mid_slot_time 110 | epoch_end_time = round(time.time()) + remaining_time 111 | time_until_next_slot = -1 112 | if leader_schedule_by_identity is not None: 113 | for slot in leader_schedule_by_identity: 114 | if current_slot_index < slot: 115 | next_slot = slot 116 | time_until_next_slot = (next_slot - current_slot_index) * mid_slot_time 117 | break 118 | else: 119 | time_until_next_slot = None 120 | 121 | result = { 122 | "epoch_endtime": epoch_end_time, 123 | "epoch_remaining_sec": remaining_time 124 | } 125 | 126 | if time_until_next_slot is not None: 127 | result.update({"time_until_next_slot": time_until_next_slot}) 128 | else: 129 | result = {} 130 | 131 | return result 132 | 133 | 134 | def get_balance_metric(balance_data, key: str): 135 | if 'value' in balance_data: 136 | result = {key: balance_data['value']} 137 | else: 138 | result = {} 139 | 140 | return result 141 | 142 | 143 | def get_solana_version_metric(solana_version_data): 144 | if solana_version_data is not None: 145 | if 'solana-core' in solana_version_data: 146 | return {'solana_version': solana_version_data['solana-core']} 147 | 148 | return {} 149 | 150 | 151 | def get_validators_metric(validators, identity_account_pubkey): 152 | if validators is not None: 153 | epoch_credits_l = [] 154 | last_vote_l = [] 155 | root_slot_l = [] 156 | current_last_vote = -1 157 | current_root_slot = -1 158 | 159 | for v in validators: 160 | if not v['delinquent']: 161 | epoch_credits_l.append(v['epochCredits']) 162 | last_vote_l.append(v['lastVote']) 163 | root_slot_l.append(v['rootSlot']) 164 | if identity_account_pubkey == v['identityPubkey']: 165 | current_last_vote = v['lastVote'] 166 | current_root_slot = v['rootSlot'] 167 | 168 | epoch_credits = np.array(epoch_credits_l, dtype=np.int32) 169 | last_vote = np.array(last_vote_l, dtype=np.int32) 170 | root_slot = np.array(root_slot_l, dtype=np.int32) 171 | 172 | last_vote = last_vote[last_vote > 0] 173 | root_slot = root_slot[root_slot > 0] 174 | 175 | cluster_max_last_vote = np.amax(last_vote) 176 | cluster_min_last_vote = np.amin(last_vote) 177 | cluster_mean_last_vote = abs((last_vote - cluster_max_last_vote).mean()) 178 | cluster_median_last_vote = abs(np.median(last_vote - cluster_max_last_vote)) 179 | 180 | cluster_max_root_slot = np.amax(root_slot) 181 | cluster_min_root_slot = np.amin(root_slot) 182 | cluster_mean_root_slot = abs((root_slot - cluster_max_root_slot).mean()) 183 | cluster_median_root_slot = abs(np.median(root_slot - cluster_max_root_slot)) 184 | 185 | result = { 186 | 'cluster_mean_epoch_credits': epoch_credits.mean(), 187 | 'cluster_min_epoch_credits': np.amin(epoch_credits), 188 | 'cluster_max_epoch_credits': np.amax(epoch_credits), 189 | 'cluster_median_epoch_credits': np.median(epoch_credits), 190 | 191 | 'cluster_max_last_vote': cluster_max_last_vote, 192 | 'cluster_min_last_vote_v2': cluster_min_last_vote, 193 | 'cluster_mean_last_vote_v2': cluster_mean_last_vote, 194 | 'cluster_median_last_vote': cluster_median_last_vote, 195 | 'current_last_vote': current_last_vote, 196 | 197 | 'cluster_max_root_slot': cluster_max_root_slot, 198 | 'cluster_min_root_slot_v2': cluster_min_root_slot, 199 | 'cluster_mean_root_slot_v2': cluster_mean_root_slot, 200 | 'cluster_median_root_slot': cluster_median_root_slot, 201 | 'current_root_slot': current_root_slot 202 | } 203 | 204 | else: 205 | result = {} 206 | 207 | return result 208 | 209 | 210 | def get_current_stake_metric(stake_data): 211 | active = 0 212 | activating = 0 213 | deactivating = 0 214 | active_cnt = 0 215 | activating_cnt = 0 216 | deactivating_cnt = 0 217 | for item in stake_data: 218 | if 'activeStake' in item: 219 | active = active + item.get('activeStake', 0) 220 | active_cnt = active_cnt + 1 221 | if 'activatingStake' in item: 222 | activating = activating + item.get('activatingStake', 0) 223 | activating_cnt = activating_cnt + 1 224 | if 'deactivatingStake' in item: 225 | deactivating = deactivating + item.get('deactivatingStake', 0) 226 | deactivating_cnt = deactivating_cnt + 1 227 | 228 | return { 229 | 'active_stake': active, 230 | 'activating_stake': activating, 231 | 'deactivating_stake': deactivating, 232 | 'stake_holders': len(stake_data), 233 | 'active_cnt': active_cnt, 234 | 'activating_cnt': activating_cnt, 235 | 'deactivating_cnt': deactivating_cnt 236 | } 237 | 238 | 239 | def load_data(config: ValidatorConfig): 240 | identity_account_pubkey = rpc.load_identity_account_pubkey(config) 241 | vote_account_pubkey = rpc.load_vote_account_pubkey(config) 242 | 243 | epoch_info_data = rpc.load_epoch_info(config) 244 | block_production_cli = rpc.load_block_production_cli(config) 245 | performance_sample_data = rpc.load_recent_performance_sample(config) 246 | solana_version_data = rpc.load_solana_version(config) 247 | validators_data = rpc.load_solana_validators(config) 248 | 249 | default = [] 250 | 251 | identity_account_balance_data = default 252 | leader_schedule_data = default 253 | block_production_data = default 254 | 255 | vote_account_balance_data = default 256 | vote_accounts_data = default 257 | stakes_data = default 258 | 259 | if identity_account_pubkey is not None: 260 | identity_account_balance_data = rpc.load_identity_account_balance(config, identity_account_pubkey) 261 | leader_schedule_data = rpc.load_leader_schedule(config, identity_account_pubkey) 262 | block_production_data = rpc.load_block_production(config, identity_account_pubkey) 263 | 264 | if vote_account_pubkey is not None: 265 | vote_account_balance_data = rpc.load_vote_account_balance(config, vote_account_pubkey) 266 | vote_accounts_data = rpc.load_vote_accounts(config, vote_account_pubkey) 267 | stakes_data = rpc.load_stakes(config, vote_account_pubkey) 268 | 269 | result = { 270 | 'identity_account_pubkey': identity_account_pubkey, 271 | 'vote_account_pubkey': vote_account_pubkey, 272 | 'identity_account_balance': identity_account_balance_data, 273 | 'vote_account_balance': vote_account_balance_data, 274 | 'epoch_info': epoch_info_data, 275 | 'leader_schedule': leader_schedule_data, 276 | 'block_production': block_production_data, 277 | 'load_block_production_cli': block_production_cli, 278 | 'vote_accounts': vote_accounts_data, 279 | 'performance_sample': performance_sample_data, 280 | 'solana_version_data': solana_version_data, 281 | 'stakes_data': stakes_data, 282 | 'validators_data': validators_data, 283 | 'cpu_model': rpc.load_cpu_model(config) 284 | } 285 | 286 | debug(config, str(result)) 287 | 288 | return result 289 | 290 | 291 | def calculate_influx_fields(data): 292 | if data is None: 293 | result = {"validator_status": 0} 294 | else: 295 | identity_account_pubkey = data['identity_account_pubkey'] 296 | 297 | vote_account_metrics = get_vote_account_metrics(data['vote_accounts'], identity_account_pubkey) 298 | leader_schedule_metrics = get_leader_schedule_metrics(data['leader_schedule'], identity_account_pubkey) 299 | epoch_metrics = data['epoch_info'] 300 | block_production_metrics = get_block_production_metrics(data['block_production'], identity_account_pubkey) 301 | if identity_account_pubkey in data['leader_schedule']: 302 | leader_schedule_by_identity = data['leader_schedule'][identity_account_pubkey] 303 | else: 304 | leader_schedule_by_identity = None 305 | 306 | performance_metrics = get_performance_metrics( 307 | data['performance_sample'], epoch_metrics, leader_schedule_by_identity) 308 | 309 | result = {"validator_status": 1} 310 | result.update(vote_account_metrics) 311 | result.update(leader_schedule_metrics) 312 | result.update(epoch_metrics) 313 | result.update(block_production_metrics) 314 | result.update(performance_metrics) 315 | result.update(get_balance_metric(data['identity_account_balance'], 'identity_account_balance')) 316 | result.update(get_balance_metric(data['vote_account_balance'], 'vote_account_balance')) 317 | result.update(get_current_stake_metric(data['stakes_data'])) 318 | result.update(get_validators_metric(data['validators_data'], identity_account_pubkey)) 319 | result.update(get_block_production_cli_metrics(data['load_block_production_cli'], identity_account_pubkey)) 320 | result.update({"cpu_model": data['cpu_model']}) 321 | 322 | return result 323 | 324 | 325 | def calculate_output_data(config: ValidatorConfig): 326 | data = load_data(config) 327 | 328 | tags = { 329 | "validator_identity_pubkey": data['identity_account_pubkey'], 330 | "validator_vote_pubkey": data['vote_account_pubkey'], 331 | "validator_name": config.validator_name, 332 | "cluster_environment": config.cluster_environment 333 | } 334 | 335 | legacy_tags = { 336 | "validator_identity_pubkey": data['identity_account_pubkey'], 337 | "validator_vote_pubkey": data['vote_account_pubkey'], 338 | "validator_name": config.validator_name, 339 | } 340 | 341 | measurement = measurement_from_fields( 342 | "validators_info", 343 | calculate_influx_fields(data), 344 | tags, 345 | config, 346 | legacy_tags 347 | ) 348 | measurement.update({"cpu_model": data['cpu_model']}) 349 | if data is not None and 'solana_version_data' in data: 350 | measurement.update(get_solana_version_metric(data['solana_version_data'])) 351 | 352 | return measurement 353 | -------------------------------------------------------------------------------- /roles/monitoring/files/output_gossip.py: -------------------------------------------------------------------------------- 1 | import solana_rpc as rpc 2 | from common import ValidatorConfig 3 | from common import print_json 4 | from common import measurement_from_fields 5 | from monitoring_config import config 6 | 7 | 8 | def calculate_output_data(config: ValidatorConfig): 9 | 10 | data = rpc.load_solana_gossip(config) 11 | 12 | measurements = [] 13 | 14 | for gossip in data: 15 | measurement = measurement_from_fields("gossip", gossip, config) 16 | measurements.append(measurement) 17 | 18 | return measurements 19 | 20 | 21 | print_json(calculate_output_data(config)) 22 | 23 | -------------------------------------------------------------------------------- /roles/monitoring/files/output_tds_measurements.py: -------------------------------------------------------------------------------- 1 | from monitoring_config import config 2 | from measurement_tds_info import calculate_output_data 3 | from common import print_json 4 | 5 | print_json(calculate_output_data(config)) -------------------------------------------------------------------------------- /roles/monitoring/files/output_validator_measurements.py: -------------------------------------------------------------------------------- 1 | from monitoring_config import config 2 | from measurement_validator_info import calculate_output_data 3 | from common import print_json 4 | 5 | print_json(calculate_output_data(config)) 6 | 7 | -------------------------------------------------------------------------------- /roles/monitoring/files/output_validators.py: -------------------------------------------------------------------------------- 1 | import solana_rpc as rpc 2 | from common import ValidatorConfig 3 | from common import print_json 4 | from common import measurement_from_fields 5 | from monitoring_config import config 6 | 7 | 8 | def calculate_output_data(config: ValidatorConfig): 9 | 10 | data = rpc.load_solana_validators(config) 11 | 12 | measurements = [] 13 | 14 | for info in data: 15 | measurement = measurement_from_fields("validators", info, config) 16 | measurements.append(measurement) 17 | 18 | return measurements 19 | 20 | 21 | print_json(calculate_output_data(config)) 22 | 23 | -------------------------------------------------------------------------------- /roles/monitoring/files/output_validators_info.py: -------------------------------------------------------------------------------- 1 | import solana_rpc as rpc 2 | from common import ValidatorConfig 3 | from common import print_json 4 | from common import measurement_from_fields 5 | from monitoring_config import config 6 | 7 | 8 | def calculate_output_data(config: ValidatorConfig): 9 | 10 | data = rpc.load_solana_validators_info(config) 11 | 12 | measurements = [] 13 | 14 | for info in data: 15 | measurement = measurement_from_fields("validators-info", info, config) 16 | measurements.append(measurement) 17 | 18 | return measurements 19 | 20 | 21 | print_json(calculate_output_data(config)) 22 | 23 | -------------------------------------------------------------------------------- /roles/monitoring/files/request_utils.py: -------------------------------------------------------------------------------- 1 | from common import ValidatorConfig 2 | import subprocess 3 | import requests 4 | import json 5 | from common import debug 6 | 7 | 8 | def execute_cmd_str(config: ValidatorConfig, cmd: str, convert_to_json: bool, default=None): 9 | """ 10 | executes shell command and return string result 11 | :param default: 12 | :param config: 13 | :param convert_to_json: 14 | :param cmd: shell command 15 | :return: returns string result or None 16 | """ 17 | try: 18 | debug(config, cmd) 19 | result: str = subprocess.check_output(cmd, shell=True, stderr=subprocess.DEVNULL, timeout=10).decode().strip() 20 | 21 | if convert_to_json: 22 | result = json.loads(result) 23 | 24 | debug(config, result) 25 | 26 | return result 27 | except: 28 | return default 29 | 30 | 31 | def rpc_call(config: ValidatorConfig, address: str, method: str, params, error_result, except_result): 32 | """ 33 | calls solana rpc (https://docs.solana.com/developing/clients/jsonrpc-api) 34 | and returns result or default 35 | :param config: 36 | :param except_result: 37 | :param error_result: 38 | :param address: local or remote rpc server address 39 | :param method: rpc method 40 | :param params: rpc call parameters 41 | :return: result or default 42 | """ 43 | try: 44 | json_request = { 45 | "jsonrpc": "2.0", 46 | "id": 1, 47 | "method": method, 48 | "params": params 49 | } 50 | debug(config, json_request) 51 | debug(config, address) 52 | 53 | json_response = requests.post(address, json=json_request).json() 54 | if 'result' not in json_response: 55 | result = error_result 56 | else: 57 | result = json_response['result'] 58 | except: 59 | result = except_result 60 | 61 | debug(config, result) 62 | 63 | return result 64 | 65 | 66 | def smart_rpc_call(config: ValidatorConfig, method: str, params, default_result): 67 | """ 68 | tries to call local rpc, if it fails tries to call remote rpc 69 | """ 70 | result = rpc_call(config, config.local_rpc_address, method, params, None, None) 71 | 72 | if result is None: 73 | result = rpc_call(config, config.remote_rpc_address, method, params, default_result, default_result) 74 | 75 | return result 76 | -------------------------------------------------------------------------------- /roles/monitoring/files/solana_rpc.py: -------------------------------------------------------------------------------- 1 | from common import ValidatorConfig 2 | from typing import Optional 3 | from common import debug 4 | from request_utils import execute_cmd_str, smart_rpc_call, rpc_call 5 | 6 | 7 | def load_identity_account_pubkey(config: ValidatorConfig) -> Optional[str]: 8 | """ 9 | loads validator identity account pubkey 10 | :param config: Validator Configuration 11 | :return: returns validator identity pubkey or None 12 | """ 13 | identity_cmd = f'solana address -u localhost --keypair ' + config.secrets_path + '/validator-keypair.json' 14 | debug(config, identity_cmd) 15 | return execute_cmd_str(config, identity_cmd, convert_to_json=False) 16 | 17 | 18 | def load_vote_account_pubkey(config: ValidatorConfig) -> Optional[str]: 19 | """ 20 | loads vote account pubkey 21 | :param config: Validator Configuration 22 | :return: returns vote account pubkey or None 23 | """ 24 | vote_pubkey_cmd = f'solana address -u localhost --keypair ' + config.secrets_path + '/vote-account-keypair.json' 25 | debug(config, vote_pubkey_cmd) 26 | return execute_cmd_str(config, vote_pubkey_cmd, convert_to_json=False) 27 | 28 | 29 | def load_vote_account_balance(config: ValidatorConfig, vote_account_pubkey: str): 30 | """ 31 | loads vote account balance 32 | https://docs.solana.com/developing/clients/jsonrpc-api#getbalance 33 | """ 34 | return smart_rpc_call(config, "getBalance", [vote_account_pubkey], {}) 35 | 36 | 37 | def load_identity_account_balance(config: ValidatorConfig, identity_account_pubkey: str): 38 | """ 39 | loads identity account balance 40 | https://docs.solana.com/developing/clients/jsonrpc-api#getbalance 41 | """ 42 | return smart_rpc_call(config, "getBalance", [identity_account_pubkey], {}) 43 | 44 | 45 | def load_epoch_info(config: ValidatorConfig): 46 | """ 47 | loads epoch info 48 | https://docs.solana.com/developing/clients/jsonrpc-api#getepochinfo 49 | """ 50 | return smart_rpc_call(config, "getEpochInfo", [], {}) 51 | 52 | 53 | def load_leader_schedule(config: ValidatorConfig, identity_account_pubkey: str): 54 | """ 55 | loads leader schedule 56 | https://docs.solana.com/developing/clients/jsonrpc-api#getleaderschedule 57 | """ 58 | params = [ 59 | None, 60 | { 61 | 'identity': identity_account_pubkey 62 | } 63 | ] 64 | return smart_rpc_call(config, "getLeaderSchedule", params, {}) 65 | 66 | 67 | def load_block_production(config: ValidatorConfig, identity_account_pubkey: str): 68 | """ 69 | loads block production 70 | https://docs.solana.com/developing/clients/jsonrpc-api#getblockproduction 71 | """ 72 | params = [ 73 | { 74 | 'identity': identity_account_pubkey 75 | } 76 | ] 77 | return smart_rpc_call(config, "getBlockProduction", params, {}) 78 | 79 | 80 | def load_block_production_cli(config: ValidatorConfig): 81 | cmd = f'solana block-production -u l --output json-compact' 82 | return execute_cmd_str(config, cmd, convert_to_json=True, default={}) 83 | 84 | 85 | def load_vote_accounts(config: ValidatorConfig, vote_account_pubkey: str): 86 | """ 87 | loads block production 88 | https://docs.solana.com/developing/clients/jsonrpc-api#getvoteaccounts 89 | """ 90 | params = [ 91 | { 92 | 'votePubkey': vote_account_pubkey 93 | } 94 | ] 95 | return smart_rpc_call(config, "getVoteAccounts", params, {}) 96 | 97 | 98 | def load_recent_performance_sample(config: ValidatorConfig): 99 | """ 100 | loads recent performance sample 101 | https://docs.solana.com/developing/clients/jsonrpc-api#getrecentperformancesamples 102 | """ 103 | params = [1] 104 | return rpc_call(config, config.remote_rpc_address, "getRecentPerformanceSamples", params, [], []) 105 | 106 | 107 | def load_solana_version(config: ValidatorConfig): 108 | """ 109 | loads solana version 110 | https://docs.solana.com/developing/clients/jsonrpc-api#getversion 111 | """ 112 | return rpc_call(config, config.local_rpc_address, "getVersion", [], [], []) 113 | 114 | 115 | def load_stake_account_rewards(config: ValidatorConfig, stake_account): 116 | cmd = f'solana stake-account ' + stake_account + ' --num-rewards-epochs=1 --with-rewards --output json-compact' 117 | return execute_cmd_str(config, cmd, convert_to_json=True) 118 | 119 | 120 | def load_solana_validators(config: ValidatorConfig): 121 | cmd = f'solana validators -ul --output json-compact' 122 | data = execute_cmd_str(config, cmd, convert_to_json=True) 123 | 124 | if (data is not None) and ('validators' in data): 125 | return data['validators'] 126 | else: 127 | return None 128 | 129 | 130 | def load_stakes(config: ValidatorConfig, vote_account): 131 | cmd = f'solana stakes ' + vote_account + ' --output json-compact' 132 | return execute_cmd_str(config, cmd, convert_to_json=True, default=[]) 133 | 134 | 135 | def load_block_time(config: ValidatorConfig, block): 136 | """ 137 | loads solana version 138 | https://docs.solana.com/developing/clients/jsonrpc-api#getblocktime 139 | """ 140 | params = [block] 141 | return rpc_call(config, config.local_rpc_address, "getBlockTime", params, None, None) 142 | 143 | # cmd = f'solana block-time -u l ' + str(block) + ' --output json-compact' 144 | # return execute_cmd_str(cmd, convert_to_json=True) 145 | 146 | 147 | def try_to_load_current_block_info(config: ValidatorConfig): 148 | epoch_info_data = load_epoch_info(config) 149 | 150 | if epoch_info_data is not None: 151 | slot_index = epoch_info_data['slotIndex'] 152 | absolute_slot = epoch_info_data['absoluteSlot'] 153 | 154 | block_time_data = load_block_time(config, absolute_slot) 155 | 156 | if block_time_data is not None: 157 | return { 158 | 'slot_index': slot_index, 159 | 'absolute_block': absolute_slot, 160 | 'block_time': block_time_data['timestamp'] 161 | } 162 | 163 | return None 164 | 165 | 166 | def load_current_block_info(config: ValidatorConfig): 167 | result = None 168 | max_tries = 10 169 | current_try = 0 170 | while result is None and current_try < max_tries: 171 | result = try_to_load_current_block_info(config) 172 | current_try = current_try + 1 173 | 174 | return result 175 | 176 | 177 | def load_cpu_model(config: ValidatorConfig): 178 | cmd = 'cat /proc/cpuinfo | grep name| uniq' 179 | cpu_info = execute_cmd_str(config, cmd, False).split(":") 180 | cpu_model = cpu_info[1].strip() 181 | 182 | if cpu_model is not None: 183 | return cpu_model 184 | else: 185 | return 'Unknown' 186 | 187 | 188 | def load_solana_validators_full(config: ValidatorConfig): 189 | cmd = f'solana validators -ul --output json-compact' 190 | return execute_cmd_str(config, cmd, convert_to_json=True) 191 | 192 | 193 | def load_solana_validators_info(config: ValidatorConfig): 194 | cmd = f'solana validator-info get --url ' + config.remote_rpc_address + ' --output json-compact' 195 | data = execute_cmd_str(config, cmd, convert_to_json=True) 196 | return data 197 | 198 | 199 | def load_solana_gossip(config: ValidatorConfig): 200 | cmd = f'solana gossip -ul --output json-compact' 201 | return execute_cmd_str(config, cmd, convert_to_json=True) 202 | 203 | 204 | -------------------------------------------------------------------------------- /roles/monitoring/files/tds_info.py: -------------------------------------------------------------------------------- 1 | from common import ValidatorConfig 2 | import requests 3 | from common import debug 4 | 5 | 6 | def tds_rpc_call(config: ValidatorConfig, identity_account_pubkey: str): 7 | 8 | address = "https://kyc-api.vercel.app/api/validators/list?search_term=" + identity_account_pubkey 9 | 10 | try: 11 | debug(config, address) 12 | json_response = requests.get(address, timeout=5).json() 13 | if 'data' not in json_response: 14 | result = {} 15 | else: 16 | result = json_response['data'] 17 | except: 18 | result = {} 19 | 20 | debug(config, result) 21 | 22 | return result 23 | 24 | 25 | def load_tds_info(config: ValidatorConfig, identity_account_pubkey: str): 26 | tds_data = tds_rpc_call(config, identity_account_pubkey) 27 | result = {} 28 | if tds_data != [] and tds_data != {}: 29 | if 'tnCalculatedStats' in tds_data[0] and tds_data[0]['tnCalculatedStats'] is not None: 30 | result = { 31 | 'tds': tds_data[0]['tnCalculatedStats'], 32 | } 33 | if 'onboardingNumber' in tds_data[0]: 34 | result['tds']['onboardingNumber'] = tds_data[0]['onboardingNumber'] 35 | result['tds']['tdsOnboardingGroup'] = tds_data[0]['tdsOnboardingGroup'] 36 | 37 | debug(config, result) 38 | 39 | return result 40 | -------------------------------------------------------------------------------- /roles/monitoring/files/validator_monitoring.py: -------------------------------------------------------------------------------- 1 | import validator_monitoring_library as vm 2 | import json 3 | from validator_monitoring_config import config 4 | import numpy 5 | 6 | 7 | class NumpyEncoder(json.JSONEncoder): 8 | """ Special json encoder for numpy types """ 9 | def default(self, obj): 10 | if isinstance(obj, (numpy.int_, numpy.intc, numpy.intp, numpy.int8, 11 | numpy.int16, numpy.int32, numpy.int64, numpy.uint8, 12 | numpy.uint16, numpy.uint32, numpy.uint64)): 13 | return int(obj) 14 | elif isinstance(obj, (numpy.float_, numpy.float16, numpy.float32, 15 | numpy.float64)): 16 | return float(obj) 17 | elif isinstance(obj, (numpy.ndarray,)): 18 | return obj.tolist() 19 | return json.JSONEncoder.default(self, obj) 20 | 21 | 22 | def process(): 23 | influx_measurement = vm.calculate_influx_data(config) 24 | print(json.dumps(influx_measurement, cls=NumpyEncoder)) 25 | 26 | 27 | process() 28 | -------------------------------------------------------------------------------- /roles/monitoring/tasks/configure_telegraf.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Create telegraf config 4 | template: 5 | src: telegraf.conf.j2 6 | dest: /etc/telegraf/telegraf.conf 7 | mode: 0644 8 | owner: root 9 | group: root 10 | backup: yes 11 | tags: 12 | - telegraf.configure 13 | 14 | - name: Allow 'telegraf' user to run solana scripts 15 | lineinfile: 16 | path: /etc/sudoers 17 | state: present 18 | regexp: '^telegraf' 19 | line: "telegraf ALL=({{ solana_user }}:{{ solana_user }}) NOPASSWD:ALL" 20 | validate: 'visudo -cf %s' 21 | tags: 22 | - telegraf.configure 23 | 24 | - name: Reload systemd 25 | systemd: 26 | daemon_reload: yes 27 | tags: 28 | - telegraf.start 29 | 30 | - name: Enable telegraf service 31 | systemd: 32 | name: telegraf 33 | enabled: yes 34 | tags: 35 | - telegraf.start 36 | 37 | - name: Start telegraf service 38 | systemd: 39 | name: telegraf 40 | state: restarted 41 | tags: 42 | - telegraf.start 43 | - monitoring.script.library 44 | - telegraf.configure 45 | become: yes 46 | -------------------------------------------------------------------------------- /roles/monitoring/tasks/install_monitoring_script.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Delete script folder 3 | file: 4 | path: "{{ solana_home }}/monitoring" 5 | state: absent 6 | tags: 7 | monitoring.script 8 | 9 | - name: Create script folder 10 | file: 11 | path: "{{ solana_home }}/monitoring" 12 | owner: "{{ solana_user }}" 13 | group: "{{ solana_user }}" 14 | mode: 0755 15 | state: directory 16 | tags: 17 | monitoring.script 18 | 19 | - name: Install python packages 20 | pip: 21 | name: 22 | - requests 23 | - numpy 24 | state: present 25 | virtualenv: "{{ solana_home }}/monitoring" 26 | # executable: pip3 27 | tags: 28 | monitoring.script 29 | 30 | - name: ensure secrets_path exists 31 | stat: 32 | path: "{{ secrets_path }}" 33 | register: secrets_path_exists 34 | tags: 35 | monitoring.script 36 | 37 | - name: create secrets_path 38 | file: 39 | path: "{{ secrets_path }}" 40 | state: directory 41 | mode: 0755 42 | owner: "{{ solana_user }}" 43 | group: "{{ solana_user }}" 44 | when: not secrets_path_exists.stat.exists 45 | tags: 46 | monitoring.script 47 | 48 | - name: copy keys 49 | copy: 50 | src: "{{ local_secrets_path }}/{{ item }}" 51 | dest: "{{ secrets_path }}" 52 | mode: 0400 53 | owner: "{{ solana_user }}" 54 | group: "{{ solana_user }}" 55 | force: false 56 | with_items: 57 | - "validator-keypair.json" 58 | - "vote-account-keypair.json" 59 | when: not secrets_path_exists.stat.exists 60 | tags: 61 | monitoring.script 62 | 63 | - name: Upload monitoring library 64 | copy: 65 | src: "files/{{ item }}" 66 | dest: "{{ solana_home }}/monitoring/{{ item }}" 67 | owner: "{{ solana_user }}" 68 | group: "{{ solana_user }}" 69 | mode: 0644 70 | with_items: 71 | - "cluster_monitoring_library.py" 72 | - "common.py" 73 | - "measurement_validator_info.py" 74 | - "output_gossip.py" 75 | - "output_validator_measurements.py" 76 | - "output_validators_info.py" 77 | - "output_validators.py" 78 | - "request_utils.py" 79 | - "solana_rpc.py" 80 | - "validator_monitoring.py" 81 | tags: 82 | - monitoring.script 83 | - monitoring.script.library 84 | 85 | 86 | - name: Upload monitoring script starter 87 | template: 88 | src: output_starter.sh.j2 89 | dest: "{{ solana_home }}/monitoring/output_starter.sh" 90 | owner: "{{ solana_user }}" 91 | group: "{{ solana_user }}" 92 | mode: 0755 93 | tags: 94 | - monitoring.script 95 | 96 | 97 | - name: Upload config script 98 | template: 99 | src: monitoring_config.py.j2 100 | dest: "{{ solana_home }}/monitoring/monitoring_config.py" 101 | owner: "{{ solana_user }}" 102 | group: "{{ solana_user }}" 103 | mode: 0644 104 | tags: 105 | - monitoring.script 106 | 107 | -------------------------------------------------------------------------------- /roles/monitoring/tasks/install_telegraf.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Adde influx repo key on Ubuntu 4 | apt_key: 5 | url: https://repos.influxdata.com/influxdata-archive_compat.key 6 | state: present 7 | when: ansible_distribution == 'Ubuntu' 8 | tags: 9 | - telegraf.install 10 | 11 | - name: Add telegraf repo 12 | block: 13 | - name: Add {{ ansible_distribution_release | lower }} telegraf repo 14 | apt_repository: 15 | repo: deb https://repos.influxdata.com/{{ ansible_distribution | lower }} {{ ansible_distribution_release | lower }} stable 16 | state: present 17 | filename: influxdb 18 | rescue: 19 | - name: ansible_distribution_release repo failed, adding bionic telegraf repo 20 | apt_repository: 21 | repo: deb https://repos.influxdata.com/{{ ansible_distribution | lower }} bionic stable 22 | state: present 23 | filename: influxdb 24 | when: ansible_distribution == 'Ubuntu' 25 | tags: 26 | - telegraf.install 27 | 28 | - name: apt update 29 | ansible.builtin.apt: 30 | update_cache: yes 31 | when: ansible_distribution == 'Ubuntu' 32 | tags: 33 | - telegraf.install 34 | 35 | - name: Install telegraf 36 | package: 37 | name: telegraf 38 | state: present 39 | tags: 40 | - telegraf.install 41 | -------------------------------------------------------------------------------- /roles/monitoring/tasks/install_telegraf_rpc.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: Install timescale telegraf 4 | apt: 5 | deb: https://telegrafreleases.blob.core.windows.net/linux/telegraf_1.13.0~with~pg-1_amd64.deb 6 | state: present 7 | tags: 8 | - telegraf.install 9 | -------------------------------------------------------------------------------- /roles/monitoring/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install additional packages 3 | import_tasks: packages.yaml 4 | tags: 5 | - monitoring 6 | 7 | - name: Install telegraf 8 | import_tasks: install_telegraf.yaml 9 | tags: 10 | - monitoring 11 | 12 | - name: Configure telegraf 13 | import_tasks: configure_telegraf.yaml 14 | tags: 15 | - monitoring 16 | 17 | - name: Install monitoring script 18 | import_tasks: install_monitoring_script.yaml 19 | tags: 20 | - monitoring 21 | -------------------------------------------------------------------------------- /roles/monitoring/tasks/packages.yaml: -------------------------------------------------------------------------------- 1 | - name: Update apt cache 2 | apt: 3 | update_cache: yes 4 | ignore_errors: yes 5 | 6 | - name: Install additional packages 7 | apt: 8 | pkg: 9 | - gpg 10 | - gpg-agent 11 | - python3 12 | - python3-pip 13 | - python3-virtualenv 14 | - coreutils 15 | tags: 16 | - config.packages 17 | -------------------------------------------------------------------------------- /roles/monitoring/templates/monitoring_config.py.j2: -------------------------------------------------------------------------------- 1 | from common import ValidatorConfig 2 | 3 | config = ValidatorConfig( 4 | validator_name="{{ validator_name }}", 5 | secrets_path="{{ secrets_path }}", 6 | local_rpc_address="http://localhost:8899", 7 | remote_rpc_address="{{ cluster_rpc_address }}", 8 | cluster_environment="{{ cluster_environment }}", 9 | debug_mode=False 10 | ) 11 | -------------------------------------------------------------------------------- /roles/monitoring/templates/output_starter.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source "{{ solana_home }}/monitoring/bin/activate" 3 | result=$(timeout -k 50 45 python3 "{{ solana_home }}/monitoring/$1.py") 4 | 5 | if [ -z "${result}" ] 6 | then 7 | echo "{}" 8 | else 9 | echo "$result" 10 | fi 11 | 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /roles/monitoring/templates/telegraf.conf.j2: -------------------------------------------------------------------------------- 1 | [agent] 2 | hostname = "{{ validator_name }}" # set this to a name you want to identify your node in the grafana dashboard 3 | flush_interval = "30s" 4 | interval = "30s" 5 | 6 | ##INPUTS 7 | [[inputs.cpu]] 8 | ## Whether to report per-cpu stats or not 9 | percpu = false 10 | ## Whether to report total system cpu stats or not 11 | totalcpu = true 12 | ## If true, collect raw CPU time metrics. 13 | collect_cpu_time = false 14 | ## If true, compute and report the sum of all non-idle CPU states. 15 | report_active = false 16 | 17 | [[inputs.disk]] 18 | ## By default stats will be gathered for all mount points. 19 | ## Set mount_points will restrict the stats to only the specified mount points. 20 | mount_points = ["/", "{{ ledger_path }}", "{{ accounts_path }}"] 21 | 22 | ## Ignore mount points by filesystem type. 23 | ignore_fs = ["devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] 24 | 25 | [[inputs.diskio]] 26 | 27 | [[inputs.net]] 28 | 29 | [[inputs.nstat]] 30 | 31 | [[inputs.procstat]] 32 | pattern="{{ solana_user }}" 33 | 34 | [[inputs.system]] 35 | 36 | [[inputs.systemd_units]] 37 | [inputs.systemd_units.tagpass] 38 | name = ["solana*"] 39 | 40 | [[inputs.mem]] 41 | 42 | [[inputs.swap]] 43 | 44 | [[inputs.exec]] 45 | commands = [ 46 | "sudo -i -u {{ solana_user }} {{ solana_home }}/monitoring/output_starter.sh output_validator_measurements" 47 | ] 48 | interval = "30s" 49 | timeout = "30s" 50 | json_name_key = "measurement" 51 | json_time_key = "time" 52 | tag_keys = ["tags_validator_name", 53 | "tags_validator_identity_pubkey", 54 | "tags_validator_vote_pubkey", 55 | "tags_cluster_environment", 56 | "validator_id", 57 | "validator_name"] 58 | 59 | json_string_fields = [ 60 | "monitoring_version", 61 | "solana_version", 62 | "validator_identity_pubkey", 63 | "validator_vote_pubkey", 64 | "cluster_environment", 65 | "cpu_model"] 66 | 67 | json_time_format = "unix_ms" 68 | 69 | ##OUTPUTS 70 | [[outputs.influxdb]] 71 | database = "{{ telegraf_database }}" 72 | urls = [ "{{ telegraf_urls }}" ] 73 | username = "{{ telegraf_username }}" 74 | password = "{{ telegraf_password }}" 75 | -------------------------------------------------------------------------------- /roles/monitoring/templates/telegraf.rpc.conf.j2: -------------------------------------------------------------------------------- 1 | [agent] 2 | hostname = "{{ validator_name }}" # set this to a name you want to identify your node in the grafana dashboard 3 | flush_interval = "30s" 4 | interval = "30s" 5 | # collection_jitter = "10s" 6 | flush_jitter = "10s" 7 | metric_buffer_limit = 10000 8 | metric_batch_size = 10000 9 | 10 | 11 | [[inputs.exec]] 12 | commands = [ 13 | "sudo -i -u {{ solana_user }} {{ solana_home }}/monitoring/output_starter.sh output_validators" 14 | ] 15 | interval = "30s" 16 | timeout = "30s" 17 | json_name_key = "measurement" 18 | json_time_key = "time" 19 | tag_keys = ["tags_cluster_environment", 20 | "tags_fields_identityPubkey", 21 | "tags_fields_voteAccountPubkey"] 22 | 23 | json_string_fields = ["solana_version", 24 | "monitoring_version", 25 | "data_version", 26 | "delinquent"] 27 | 28 | json_time_format = "unix_ms" 29 | 30 | [[inputs.exec]] 31 | commands = [ 32 | "sudo -i -u {{ solana_user }} {{ solana_home }}/monitoring/output_starter.sh output_gossip" 33 | ] 34 | interval = "3600s" 35 | timeout = "3600s" 36 | json_name_key = "measurement" 37 | json_time_key = "time" 38 | tag_keys = ["fields_identityPubkey", "cluster_environment"] 39 | 40 | json_string_fields = ["solana_version", 41 | "monitoring_version", 42 | "data_version", 43 | "data_ipAddress"] 44 | 45 | json_time_format = "unix_ms" 46 | 47 | 48 | [[inputs.exec]] 49 | commands = [ 50 | "sudo -i -u {{ solana_user }} {{ solana_home }}/monitoring/output_starter.sh output_validators_info" 51 | ] 52 | interval = "120s" 53 | timeout = "120s" 54 | json_name_key = "measurement" 55 | json_time_key = "time" 56 | tag_keys = ["data_identityPubkey", "cluster_environment"] 57 | 58 | json_string_fields = ["monitoring_version", 59 | "data_infoPubkey", 60 | "data_ipAddress", 61 | "data_info_details", 62 | "data_info_keybaseUsername", 63 | "data_info_name", 64 | "data_info_website"] 65 | 66 | json_time_format = "unix_ms" 67 | 68 | ##OUPUTS TimeScale 69 | [[outputs.postgresql]] 70 | connection = "host={{ ts_host }} user=tsdbadmin password={{ ts_password }} sslmode=require dbname={{ ts_dbname }}" 71 | table_template="CREATE TABLE IF NOT EXISTS {TABLE}({COLUMNS}); SELECT create_hypertable({TABLELITERAL},'time',chunk_time_interval := INTERVAL '1 day',if_not_exists := true);" 72 | 73 | -------------------------------------------------------------------------------- /roles/restart_cluster/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: stop solana validator service 4 | systemd: 5 | name: solana-validator 6 | state: stopped 7 | tags: 8 | - validator.cluster.restart 9 | - validator.cluster.restart.stop 10 | 11 | - name: install solana 1.7.12 12 | shell: solana-install init 1.7.12 13 | environment: 14 | PATH: "{{ solana_home }}/.local/share/solana/install/active_release/bin" 15 | become: yes 16 | become_user: "{{ solana_user }}" 17 | tags: 18 | - validator.cluster.restart 19 | - validator.cluster.restart.cli.install 20 | 21 | - name: create snapshot from ledger 22 | become: yes 23 | become_user: "{{ solana_user }}" 24 | shell: "solana-ledger-tool --ledger {{ ledger_path }} create-snapshot 95038710 {{ snapshots_path }} --snapshot-archive-path {{ snapshots_path }} --hard-fork 95038710 --wal-recovery-mode skip_any_corrupted_record" 25 | environment: 26 | #PATH: "{{ node.solana_home }}/.local/share/solana/install/active_release/bin" 27 | PATH: "/home/solana/.local/share/solana/install/active_release/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin" 28 | HOME: "/home/solana" 29 | tags: 30 | - validator.cluster.restart 31 | - validator.cluster.restart.ledger.snapshot 32 | 33 | - name: backup solana validator service 34 | shell: "cp /etc/systemd/system/solana-validator.service /etc/systemd/system/solana-validator.service.backup.220921" 35 | tags: 36 | - validator.cluster.restart 37 | - validator.cluster.restart.backup 38 | 39 | - name: Create solana validator service 40 | template: 41 | src: solana-validator.{{ cluster_environment }}.restart.service.j2 42 | dest: /etc/systemd/system/solana-validator.service 43 | mode: 0644 44 | owner: root 45 | group: root 46 | tags: 47 | - validator.cluster.restart 48 | - validator.cluster.restart.service 49 | 50 | - name: start solana validator service 51 | systemd: 52 | name: solana-validator 53 | state: reloaded 54 | daemon_reload: yes 55 | tags: 56 | - validator.cluster.restart 57 | - validator.cluster.restart.start 58 | 59 | -------------------------------------------------------------------------------- /roles/restart_cluster/templates/create-snapshot.sh.j2: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #set -x -e 3 | 4 | create_snapshot_form_ledger() { 5 | sudo -i -u solana solana-ledger-tool --ledger {{ validator.ledger_path }} create-snapshot 95038710 {{ validator.snapshots_path }} --snapshot-archive-path {{ validator.snapshots_path }} --hard-fork 95038710 --wal-recovery-mode skip_any_corrupted_record 6 | } 7 | 8 | create_snapshot_form_ledger 9 | -------------------------------------------------------------------------------- /roles/restart_cluster/templates/solana-validator.testnet.restart.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Solana {{ cluster_environment }} node 3 | After=network.target syslog.target 4 | StartLimitIntervalSec=0 5 | 6 | [Service] 7 | Type=simple 8 | Restart=always 9 | RestartSec=1 10 | User={{ solana_user }} 11 | LimitNOFILE=1024000 12 | Environment="PATH=/bin:/usr/bin:{{ env_path }}" 13 | Environment="SOLANA_METRICS_CONFIG=host=https://metrics.solana.com:8086,db=tds,u=testnet_write,p=c4fa841aa918bf8274e3e2a44d77568d9861b3ea" 14 | Environment="RUST_LOG={{ log_level }}" 15 | ExecStart={{ env_path }}/solana-validator \ 16 | --identity {{ secrets_path }}/validator-keypair.json \ 17 | --vote-account {{ secrets_path }}/vote-account-keypair.json \ 18 | --rpc-port {{ solana_rpc_port }} \ 19 | --wait-for-supermajority 95038710 \ 20 | --no-snapshot-fetch \ 21 | --no-genesis-fetch \ 22 | --expected-bank-hash E3MJucWkWkugqJ8ewHAkDWuCN6uDxEychwjFFCwJ16ic \ 23 | --expected-shred-version 50850 \ 24 | --known-validator 3K8BYGTPD9AxqYQDPdU8PPy6AfiSwf4hDmFy1xXGB8Ns \ 25 | --known-validator 5dB4Ygb8Sf3Sssdxxrpbb4NFX9bMrYnieiz11Vr5xJkJ \ 26 | --known-validator 7TcmJn12spW6KQJp4fvvo45d1hpxS8EnLjKMxihtNZ1V \ 27 | --known-validator 4jhyvbBHbsRDF6och7pDQ7ahYTUr7wNkAYJTLLuMUtku \ 28 | --known-validator 4rVaXrd7BLSFZMSm4Lq63nxkVyezGxsQVpUhc9LqbxVk \ 29 | --known-validator 82k4RGZAJxtXvW3hzgmHB2q4oDHzgwMR2cGXup324gsJ \ 30 | {% for entrypoint in entrypoints %} 31 | --entrypoint {{ entrypoint }} \ 32 | {% endfor %} 33 | --no-port-check \ 34 | --limit-ledger-size \ 35 | --log {{ validator_log_path }} \ 36 | --ledger {{ ledger_path }} \ 37 | --accounts {{ accounts_path }} \ 38 | --accounts-db-caching-enabled \ 39 | --snapshots {{ snapshots_path }} \ 40 | --dynamic-port-range {{ open_solana_ports_start }}-{{ open_solana_ports_end }} 41 | 42 | 43 | [Install] 44 | WantedBy=multi-user.target 45 | -------------------------------------------------------------------------------- /roles/solana_cli/tasks/install.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: create download dir 3 | file: 4 | path: /tmp/solana 5 | state: directory 6 | owner: "{{ solana_user }}" 7 | group: "{{ solana_user }}" 8 | tags: 9 | - cli.install 10 | 11 | - name: install solana 12 | block: 13 | - name: download latest solana release installer 14 | get_url: 15 | url: "https://release.solana.com/v{{ solana_version | default('stable') }}/install" 16 | dest: /tmp/solana/ 17 | mode: 0755 18 | 19 | - name: run solana installer 20 | shell: /tmp/solana/install 21 | become: yes 22 | become_user: "{{ solana_user }}" 23 | tags: 24 | - cli.install 25 | when: jito is not defined or not jito 26 | 27 | - name: install JITO solana 28 | block: 29 | - name: download latest jito release installer 30 | get_url: 31 | url: "https://release.jito.wtf/v{{ jito_version | default('stable') }}-jito/install" 32 | dest: /tmp/solana/ 33 | mode: 0755 34 | 35 | - name: run jito installer 36 | shell: /tmp/solana/install 37 | become: yes 38 | become_user: "{{ solana_user }}" 39 | tags: 40 | - cli.install 41 | when: jito is defined and jito 42 | 43 | - name: remove installer 44 | file: 45 | path: /tmp/solana 46 | state: absent 47 | tags: 48 | - cli.install 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /roles/solana_cli/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: set force install fact 3 | set_fact: 4 | force: "{{ force | default('false') }}" 5 | tags: 6 | - cli 7 | - cli.install 8 | - cli.update 9 | 10 | - name: check solana cli installed 11 | stat: 12 | path: /home/solana/.local/share/solana/install/active_release/bin/solana-install 13 | register: solana_exists 14 | tags: 15 | - cli 16 | - cli.install 17 | - cli.update 18 | 19 | - name: install solana cli 20 | import_tasks: install.yaml 21 | tags: 22 | - cli 23 | - cli.install 24 | when: force == 'true' or not solana_exists.stat.exists 25 | 26 | - name: update solana cli 27 | import_tasks: update.yaml 28 | tags: 29 | - cli 30 | - cli.update 31 | when: force != 'true' and solana_exists.stat.exists 32 | -------------------------------------------------------------------------------- /roles/solana_cli/tasks/update.yaml: -------------------------------------------------------------------------------- 1 | - name: DEBUG 2 | debug: 3 | msg: "Updating Solana to {{ solana_version }}" 4 | 5 | - name: update solana 6 | shell: "solana-install init {{ solana_version }}" 7 | become: yes 8 | become_user: "{{ solana_user }}" 9 | environment: 10 | PATH: "{{ solana_home }}/.local/share/solana/install/active_release/bin" 11 | tags: 12 | - cli.update 13 | -------------------------------------------------------------------------------- /roles/solana_config/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: ensure directory exists 4 | file: 5 | path: "/etc/sv_manager" 6 | state: directory 7 | mode: '0755' 8 | 9 | - name: Check if config file exists 10 | stat: 11 | path: /etc/sv_manager/sv_manager.conf 12 | register: config_exists 13 | 14 | - name: Load config file 15 | slurp: 16 | src: /etc/sv_manager/sv_manager.conf 17 | register: config_data 18 | when: config_exists.stat.exists 19 | 20 | # - name: show config data 21 | # debug: 22 | # msg: "{{ config_data }}" 23 | # when: config_exists.stat.exists 24 | 25 | - name: set dictionary 26 | set_fact: 27 | config_dict: "{{ config_data.content | default([]) }}" 28 | when: not config_exists.stat.exists 29 | 30 | - name: dictionary 31 | set_fact: 32 | config_dict: "{{ config_data.content | b64decode | from_json | default([]) }}" 33 | when: config_exists.stat.exists 34 | 35 | - name: create config from template 36 | set_fact: 37 | config_dict_template: "{{ lookup('template', 'templates/sv_manager.conf.template') }}" 38 | 39 | - name: show present config dictionary 40 | debug: 41 | msg: "{{ config_dict }}" 42 | 43 | 44 | - name: combine with host section 45 | set_fact: 46 | config_dict: "{{ config_dict_template | combine(config_dict) }}" 47 | 48 | 49 | - name: show new config 50 | debug: 51 | var: config_dict 52 | 53 | - name: write var to file 54 | copy: 55 | content: "{{ config_dict | to_nice_json }}" 56 | dest: /etc/sv_manager/sv_manager.conf 57 | -------------------------------------------------------------------------------- /roles/solana_config/templates/sv_manager.conf.template: -------------------------------------------------------------------------------- 1 | { 2 | "cluster_environment": "{{ hostvars[inventory_hostname]["cluster_environment"] }}", 3 | "cluster_rpc_address": "{{ hostvars[inventory_hostname]["cluster_rpc_address"] }}", 4 | "flat_path": "{{ hostvars[inventory_hostname]["flat_path"] }}", 5 | "keybase_username": "{{ hostvars[inventory_hostname]["keybase_username"] }}", 6 | "ledger_path": "{{ hostvars[inventory_hostname]["ledger_path"] }}", 7 | "local_secrets_path": "{{ hostvars[inventory_hostname]["local_secrets_path"] }}", 8 | "log_level": "{{ hostvars[inventory_hostname]["log_level"] }}", 9 | "lvm_enabled": "{{ hostvars[inventory_hostname]["lvm_enabled"] }}", 10 | "lvm_vg": "{{ hostvars[inventory_hostname]["lvm_vg"] | default(omit) }}", 11 | "ramdisk_size_gb": "{{ hostvars[inventory_hostname]["ramdisk_size_gb"] }}", 12 | "secrets_path": "{{ hostvars[inventory_hostname]["secrets_path"] }}", 13 | "set_validator_info": "{{ hostvars[inventory_hostname]["set_validator_info"] }}", 14 | "snapshots_path": "{{ hostvars[inventory_hostname]["snapshots_path"] }}", 15 | "solana_user": "{{ hostvars[inventory_hostname]["solana_user"] }}", 16 | "solana_validator_service": "{{ hostvars[inventory_hostname]["solana_validator_service"] }}", 17 | "solana_version": "{{ hostvars[inventory_hostname]["solana_version"] }}", 18 | "swap_file_size_gb": "{{ hostvars[inventory_hostname]["swap_file_size_gb"] }}", 19 | "upload_validator_keys": "{{ hostvars[inventory_hostname]["upload_validator_keys"] }}", 20 | "validator_homepage": "{{ hostvars[inventory_hostname]["validator_homepage"] }}", 21 | "validator_name": "{{ hostvars[inventory_hostname]["validator_name"] }}" 22 | } -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/cluster_environment.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: set cluster environment 3 | shell: solana config set --url {{ cluster_rpc_address }} 4 | become: yes 5 | become_user: "{{ solana_user }}" 6 | environment: 7 | PATH: "{{ env_path }}" 8 | tags: 9 | validator.manage.cluster 10 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/configure_validator.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: set default keypair 3 | shell: solana config set --keypair {{ secrets_path }}/validator-keypair.json 4 | become: yes 5 | become_user: "{{ solana_user}}" 6 | environment: 7 | PATH: "{{ env_path }}" 8 | tags: 9 | validator.manage.config 10 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/create_missing_keys.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Create keys: Check identity keypair exists" 3 | stat: 4 | path: "{{ secrets_path }}/validator-keypair.json" 5 | register: validator_keypair_exists 6 | tags: 7 | - validator.manage.keys.create.identity 8 | 9 | - name: "Create keys: Check vote account keypair exists" 10 | stat: 11 | path: "{{ secrets_path }}/vote-account-keypair.json" 12 | register: vote_account_keypair_exists 13 | become: yes 14 | tags: 15 | - validator.manage.keys.create.vote 16 | 17 | - name: "Create keys: Build flat keys path" 18 | set_fact: 19 | secrets_local_path: "{{ local_secrets_path }}" 20 | when: flat_path|bool 21 | tags: 22 | - validator.manage.keys.create 23 | 24 | - name: "Create keys: Build non flat keys path" 25 | set_fact: 26 | secrets_local_path: "{{ local_secrets_path }}/{{ validator_name }}/solana/" 27 | when: not flat_path|bool 28 | tags: 29 | - validator.manage.keys.create 30 | 31 | - name: "Create keys: Check identity keypair exists locally" 32 | connection: local 33 | become: no 34 | delegate_to: localhost 35 | stat: 36 | path: "{{ secrets_local_path }}/validator-keypair.json" 37 | register: validator_keypair_exists_locally 38 | tags: 39 | - validator.manage.keys.create.identity 40 | 41 | - name: "Create keys: Check vote account keypair exists locally" 42 | connection: local 43 | become: no 44 | delegate_to: localhost 45 | stat: 46 | path: "{{ secrets_local_path }}/vote-account-keypair.json" 47 | register: vote_account_keypair_exists_locally 48 | tags: 49 | - validator.manage.keys.create.vote 50 | 51 | - name: "Create keys: Create identity keypair" 52 | shell: "solana-keygen new --silent --no-bip39-passphrase --outfile {{ secrets_path }}/validator-keypair.json" 53 | environment: 54 | PATH: "{{ env_path }}" 55 | tags: 56 | validator.manage.keys.create.identity 57 | when: not validator_keypair_exists_locally.stat.exists 58 | and not validator_keypair_exists.stat.exists 59 | and (cluster_environment == 'testnet' or rpc_node) 60 | 61 | - name: "Create keys: Create vote account keypair" 62 | shell: "solana-keygen new --silent --no-bip39-passphrase --outfile {{ secrets_path }}/vote-account-keypair.json" 63 | environment: 64 | PATH: "{{ env_path }}" 65 | tags: 66 | validator.manage.keys.create.vote-account 67 | when: not vote_account_keypair_exists_locally.stat.exists 68 | and not vote_account_keypair_exists.stat.exists 69 | and cluster_environment == 'testnet' 70 | and not rpc_node 71 | 72 | - name: "Create keys: Airdrop 1 sol to validator-keypair" 73 | shell: "solana airdrop --keypair {{ secrets_path }}/validator-keypair.json 1" 74 | environment: 75 | PATH: "{{ env_path }}" 76 | tags: 77 | validator.manage.keys.create.airdrop 78 | when: not vote_account_keypair_exists.stat.exists 79 | and cluster_environment == 'testnet' 80 | 81 | - name: "Create keys: Create vote account" 82 | shell: "solana create-vote-account {{ secrets_path }}/vote-account-keypair.json {{ secrets_path }}/validator-keypair.json --keypair ~/.secrets/validator-keypair.json" 83 | environment: 84 | PATH: "{{ env_path }}" 85 | tags: 86 | validator.manage.keys.create.airdrop 87 | when: not vote_account_keypair_exists.stat.exists 88 | and not vote_account_keypair_exists_locally.stat.exists 89 | and cluster_environment == 'testnet' 90 | and not rpc_node 91 | 92 | - name: "Create keys: fetch validator-keypair" 93 | fetch: 94 | src: "{{ secrets_path }}/validator-keypair.json" 95 | dest: "{{ secrets_local_path }}" 96 | flat: true 97 | tags: 98 | validator.manage.keys.create.fetch 99 | when: not validator_keypair_exists.stat.exists 100 | and not validator_keypair_exists_locally.stat.exists 101 | and "'local' not in group_names" 102 | and not rpc_node 103 | 104 | - name: "Create keys: fetch vote-account-keypair" 105 | fetch: 106 | src: "/home/{{ solana_user }}/.secrets/vote-account-keypair.json" 107 | dest: "{{ secrets_local_path }}" 108 | flat: true 109 | tags: 110 | validator.manage.keys.create.fetch 111 | when: not vote_account_keypair_exists.stat.exists 112 | and not vote_account_keypair_exists_locally.stat.exists 113 | and "'local' not in group_names" 114 | and not rpc_node 115 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/logrotate.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create log file dir 3 | file: 4 | path: "{{ validator_log_path }}" 5 | state: directory 6 | owner: solana 7 | group: users 8 | mode: '0755' 9 | 10 | - name: Create solana logrotate 11 | template: 12 | src: solana-validator.logrotate.j2 13 | dest: /etc/logrotate.d/solana-validator.logrotate 14 | mode: 0644 15 | owner: root 16 | group: root 17 | tags: 18 | validator.logrotate 19 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Set cluster environment 3 | import_tasks: cluster_environment.yaml 4 | tags: 5 | - validator.manage.cluster 6 | 7 | - name: Upload keys 8 | import_tasks: upload_keys.yaml 9 | tags: 10 | - validator.manage.keys 11 | when: upload_validator_keys 12 | and not rpc_node 13 | 14 | - name: Create missing keys 15 | import_tasks: create_missing_keys.yaml 16 | become: yes 17 | become_user: "{{ solana_user }}" 18 | tags: 19 | - validator.manage.keys 20 | 21 | - name: Confugure validator 22 | import_tasks: configure_validator.yaml 23 | tags: 24 | - validator.manage.config 25 | 26 | - name: Create sys-tuner service 27 | import_tasks: sys-tuner.service.yaml 28 | tags: 29 | - validator.service.sys-tuner 30 | 31 | - name: Setup logrotate 32 | import_tasks: logrotate.yaml 33 | tags: 34 | - validator.logrotate 35 | 36 | - name: Create solana validator service 37 | import_tasks: solana-validator.service.yaml 38 | tags: 39 | - validator.service.solana 40 | 41 | # - name: Set validator info 42 | # import_tasks: validator_info.yaml 43 | # tags: 44 | # - validator.manage.info 45 | # when: set_validator_info 46 | 47 | 48 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/solana-validator.service.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Remove old service file 3 | file: 4 | path: /etc/systemd/system/solana-validator.service 5 | state: absent 6 | tags: 7 | - validator.service.solana 8 | 9 | - name: Create solana validator service 10 | template: 11 | src: solana-validator.service.j2 12 | dest: /lib/systemd/system/solana-validator.service 13 | mode: 0644 14 | owner: root 15 | group: root 16 | tags: 17 | - validator.service.solana 18 | 19 | - name: Reload systemd 20 | systemd: 21 | daemon_reload: yes 22 | tags: 23 | - validator.service.solana 24 | 25 | - name: Enable solana service 26 | systemd: 27 | name: solana-validator 28 | enabled: yes 29 | tags: 30 | - validator.service.solana 31 | 32 | - name: Start solana service 33 | systemd: 34 | name: solana-validator 35 | state: "{{ solana_validator_service }}" 36 | tags: 37 | - validator.service.solana 38 | 39 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/sys-tuner.service.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create solana-sys-tuner service 3 | template: 4 | src: solana-sys-tuner.service.j2 5 | dest: /lib/systemd/system/solana-sys-tuner.service 6 | mode: 0644 7 | owner: root 8 | group: root 9 | tags: 10 | validator.service.sys-tuner 11 | 12 | - name: Reload service configuration 13 | systemd: 14 | daemon_reload: yes 15 | tags: 16 | validator.service.sys-tuner 17 | 18 | - name: Enable solana-sys-tuner service 19 | systemd: 20 | name: solana-sys-tuner 21 | enabled: yes 22 | tags: 23 | validator.service.sys-tuner 24 | 25 | - name: Restart solana-sys-tuner service 26 | systemd: 27 | name: solana-sys-tuner 28 | state: restarted 29 | tags: 30 | validator.service.sys-tuner 31 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/upload_keys.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: "Upload keys: Build flat keys path" 3 | set_fact: 4 | secrets_local_path: "{{ local_secrets_path }}" 5 | when: flat_path|bool 6 | 7 | - name: "Upload keys: Build non flat keys path" 8 | set_fact: 9 | secrets_local_path: "{{ local_secrets_path }}/{{ validator_name }}/solana" 10 | when: not flat_path|bool 11 | 12 | - name: "Upload keys: Check validator-keypair exists locally" 13 | stat: 14 | path: "{{ secrets_local_path }}/validator-keypair.json" 15 | connection: local 16 | become: no 17 | delegate_to: localhost 18 | register: validator_keypair_exists 19 | tags: 20 | - validator.manage.keys.create.vote 21 | when: not fail_if_no_validator_keypair 22 | 23 | - name: "Upload keys: upload validator-keypair" 24 | copy: 25 | src: "{{ secrets_local_path }}/validator-keypair.json" 26 | dest: "{{ secrets_path }}" 27 | mode: 0400 28 | owner: "{{ solana_user }}" 29 | group: "{{ solana_user }}" 30 | force: false 31 | tags: 32 | - validator.manage.keys.upload 33 | when: fail_if_no_validator_keypair 34 | or validator_keypair_exists.stat.exists 35 | 36 | - name: "Upload keys: Check vote account keypair exists locally" 37 | stat: 38 | path: "{{ secrets_local_path }}/vote-account-keypair.json" 39 | connection: local 40 | become: no 41 | delegate_to: localhost 42 | register: vote_account_keypair_exists 43 | tags: 44 | - validator.manage.keys.create.vote 45 | when: not fail_if_no_validator_keypair 46 | 47 | - name: "Upload keys: upload vote-account-keypair" 48 | copy: 49 | src: "{{ secrets_local_path }}/vote-account-keypair.json" 50 | dest: "{{ secrets_path }}" 51 | mode: 0400 52 | owner: "{{ solana_user }}" 53 | group: "{{ solana_user }}" 54 | force: false 55 | tags: 56 | - validator.manage.keys.upload 57 | when: 58 | fail_if_no_validator_keypair 59 | or vote_account_keypair_exists.stat.exists 60 | 61 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/tasks/validator_info.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Load validator pubkey 3 | shell: solana-keygen pubkey {{ secrets_path }}/validator-keypair.json 4 | register: validator_pubkey 5 | environment: 6 | PATH: "{{ env_path }}" 7 | tags: 8 | validator.manage.info 9 | 10 | - debug: var=validator_pubkey.stdout_lines[0] 11 | tags: 12 | validator.manage.info 13 | 14 | - name: Set homepage parameter 15 | set_fact: 16 | cmd_homepage_param: "-w {{ validator_homepage }}" # hosts.yaml 17 | when: validator_homepage != "" 18 | tags: 19 | validator.manage.info 20 | 21 | - name: Register keybase 22 | block: 23 | - name: Check if keybase is present 24 | uri: 25 | url: https://keybase.pub/{{ keybase_username }}/solana/validator-{{ validator_pubkey.stdout_lines[0] }} 26 | - name: set keybase-check passed shell-command 27 | set_fact: 28 | cmd_keybase_param: -n "{{ keybase_username }}" 29 | rescue: 30 | - name: Notify if keybase is skipped 31 | debug: 32 | msg: "keybase skipped" 33 | tags: 34 | validator.manage.info 35 | 36 | - name: Set compose set info command 37 | set_fact: 38 | shell_cmd: "solana validator-info publish --keypair {{ secrets_path }}/validator-keypair.json {{ cmd_keybase_param | default('') }} {{ cmd_homepage_param | default('') }} -d \"{{ validator_description }}\" {{ validator_name }}" 39 | tags: 40 | validator.manage.info 41 | 42 | - debug: 43 | var: shell_cmd 44 | tags: 45 | validator.manage.info 46 | 47 | - name: Set validator info 48 | shell: "{{ shell_cmd }}" 49 | become: yes 50 | become_user: "{{ solana_user }}" 51 | environment: 52 | PATH: "{{ env_path }}" 53 | tags: 54 | validator.manage.info 55 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/templates/service.demo: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Solana mainnet node 3 | After=network.target syslog.target 4 | StartLimitIntervalSec=0 5 | [Service] 6 | Type=simple 7 | Restart=always 8 | RestartSec=1 9 | LimitNOFILE=1024000 10 | Environment="SOLANA_METRICS_CONFIG=host=https://metrics.solana.com:8086,db=mainnet-beta,u=mainnet-beta_write,p=password" 11 | Environment="RUST_LOG=debug,hyper::proto=info,reqwest=info,hyper::client=info,solana_metrics=warn,solana_runtime=info,solana_vote_program=info,solana_core=info,solana_ledger=info,solana_rbpf=info,solana_program=info,solana_perf=info,tokio_reactor=info,jsonrpc_core=info,jsonrpc_core::io=info" 12 | ExecStart=/home/solana/.local/share/solana/install/active_release/bin/solana-validator \ 13 | --entrypoint entrypoint.mainnet-beta.solana.com:8001 \ 14 | --entrypoint entrypoint2.mainnet-beta.solana.com:8001 \ 15 | --entrypoint entrypoint3.mainnet-beta.solana.com:8001 \ 16 | --entrypoint entrypoint4.mainnet-beta.solana.com:8001 \ 17 | --entrypoint entrypoint5.mainnet-beta.solana.com:8001 \ 18 | --trusted-validator 7Np41oeYqPefeNQEHSv1UDhYrehxin3NStELsSKCT4K2 \ 19 | --trusted-validator GdnSyH3YtwcxFvQrVVJMm1JhTS4QVX7MFsX56uJLUfiZ \ 20 | --trusted-validator DE1bawNcRJB9rVm3buyMVfr8mBEoyyu73NBovf2oXJsJ \ 21 | --trusted-validator CakcnaRDHka2gXyfbEd2d3xsvkJkqsLw2akB3zsN1D2S \ 22 | --expected-genesis-hash 5eykt4UsFv8P8NJdTREpY1vzqKqZKvdpKuc147dw2N9d \ 23 | --no-untrusted-rpc \ 24 | --wal-recovery-mode skip_any_corrupted_record \ 25 | --identity /home/solana/mainnet/validator-keypair.json \ 26 | #--vote-account /home/solana/mainnet/vote-account-keypair.json \ 27 | --no-voting \ 28 | --ledger /home/solana/mainnet/ledger \ 29 | --accounts /mnt/ramdisk/mainnet/accounts \ 30 | --limit-ledger-size 50000000 \ 31 | --dynamic-port-range 8000-8010 \ 32 | --log /home/solana/mainnet/solana.log \ 33 | --snapshot-interval-slots 1000 \ 34 | --maximum-local-snapshot-age 1000 \ 35 | --snapshot-compression none \ 36 | --no-port-check \ 37 | --rpc-bind-address 127.0.0.1 \ 38 | --rpc-port 8899 \ 39 | --accounts-db-caching-enabled \ 40 | --account-index program-id spl-token-mint spl-token-owner 41 | ExecReload=/bin/kill -s HUP $MAINPID 42 | ExecStop=/bin/kill -s QUIT $MAINPID 43 | [Install] 44 | WantedBy=multi-user.target 45 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/templates/solana-sys-tuner.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Solana System Tuner Service 3 | After=network.target syslog.target 4 | 5 | [Service] 6 | Type=oneshot 7 | {% for param in sysctl_params %} 8 | ExecStart=/usr/sbin/sysctl -w {{ param }} 9 | {% endfor %} 10 | 11 | 12 | 13 | [Install] 14 | WantedBy=multi-user.target 15 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/templates/solana-validator.logrotate.j2: -------------------------------------------------------------------------------- 1 | {{ validator_log_path }}/solana-validator.log { 2 | su root root 3 | rotate 1 4 | daily 5 | size 1G 6 | compress 7 | missingok 8 | postrotate 9 | systemctl kill -s USR1 solana-validator.service 10 | endscript 11 | } 12 | 13 | -------------------------------------------------------------------------------- /roles/solana_validator_bootstrap/templates/solana-validator.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Solana {{ cluster_environment }} node 3 | After=network.target syslog.target 4 | StartLimitIntervalSec=0 5 | 6 | [Service] 7 | Type=simple 8 | Restart=always 9 | RestartSec=1 10 | User={{ solana_user }} 11 | LimitNOFILE=1000000 12 | Environment="PATH=/bin:/usr/bin:{{ env_path }}" 13 | Environment="SOLANA_METRICS_CONFIG=host={{ solana_metrics_url }}" 14 | {% if agave is defined and agave %} 15 | ExecStart={{ env_path }}/agave-validator \ 16 | {% else %} 17 | ExecStart={{ env_path }}/solana-validator \ 18 | {% endif %} 19 | --identity {{ secrets_path }}/validator-keypair.json \ 20 | {% if not rpc_node %} 21 | --vote-account {{ secrets_path }}/vote-account-keypair.json \ 22 | {% endif %} 23 | --rpc-port {{ solana_rpc_port }} \ 24 | --full-rpc-api \ 25 | --only-known-rpc \ 26 | --expected-genesis-hash {{ expected_genesis_hash }} \ 27 | --incremental-snapshot-interval-slots {{ incremental_snapshot_interval_slots }} \ 28 | --accounts-hash-interval-slots {{ incremental_snapshot_interval_slots }} \ 29 | --use-snapshot-archives-at-startup when-newest \ 30 | {% if not extra_params is none %} 31 | {% for extra_var in extra_params %} 32 | {{ extra_var }} \ 33 | {% endfor %} 34 | {% endif %} 35 | {% for entrypoint in entrypoints %} 36 | --entrypoint {{ entrypoint }} \ 37 | {% endfor %} 38 | {% for known_validator in known_validators %} 39 | --known-validator {{ known_validator }} \ 40 | {% endfor %} 41 | {%if 'mainnet_validators' in group_names and not rpc_node %} 42 | --private-rpc \ 43 | --wal-recovery-mode skip_any_corrupted_record \ 44 | --rpc-bind-address 127.0.0.1 \ 45 | {% endif %} 46 | {% if rpc_node %} 47 | --no-voting \ 48 | --account-index program-id \ 49 | --account-index spl-token-mint \ 50 | --account-index spl-token-owner \ 51 | {% endif %} 52 | --limit-ledger-size {{ limit_ledger_size }} \ 53 | --log {{ validator_log_path }}/solana-validator.log \ 54 | --ledger {{ ledger_path }} \ 55 | --accounts {{ accounts_path }} \ 56 | --snapshots {{ snapshots_path }} \ 57 | {% if jito is defined and jito %} 58 | --relayer-url http://frankfurt.mainnet.relayer.jito.wtf:8100 \ 59 | --block-engine-url https://frankfurt.mainnet.block-engine.jito.wtf \ 60 | --shred-receiver-address 145.40.93.84:1002 \ 61 | --tip-payment-program-pubkey T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt \ 62 | --tip-distribution-program-pubkey 4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7 \ 63 | --merkle-root-upload-authority GZctHpWXmsZC1YHACTGGcHhYxjdRqQvTpYkb9LMvxDib \ 64 | --commission-bps 800 \ 65 | {% endif %} 66 | --dynamic-port-range {{ open_solana_ports_start }}-{{ open_solana_ports_end }} 67 | 68 | 69 | [Install] 70 | WantedBy=multi-user.target 71 | -------------------------------------------------------------------------------- /roles/solana_validator_restart/tasks/main.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: wait for restarting window 4 | import_tasks: wait_for_restart_window.yaml 5 | tags: 6 | - validator.manage.restart 7 | 8 | - name: restart sys_tuner-service 9 | systemd: 10 | name: solana-sys-tuner 11 | state: restarted 12 | tags: 13 | - validator.manage.restart.service 14 | - validator.manage.restart 15 | 16 | - name: restart solana validator service 17 | systemd: 18 | name: solana-validator 19 | state: restarted 20 | tags: 21 | - validator.manage.restart.service 22 | - validator.manage.restart 23 | 24 | 25 | -------------------------------------------------------------------------------- /roles/solana_validator_restart/tasks/wait_for_restart_window.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | - name: wait for restarting window 4 | become: yes 5 | become_user: "{{ solana_user }}" 6 | shell: solana-validator --ledger {{ ledger_path }} wait-for-restart-window --max-delinquent-stake {{ max_delinquent_stake }} 7 | environment: 8 | PATH: "{{ env_path }}" 9 | ignore_errors: True 10 | when: wait_for_window and agave is not defined 11 | tags: 12 | - validator.manage.restart.wait 13 | 14 | - name: wait for restarting window 15 | become: yes 16 | become_user: "{{ solana_user }}" 17 | shell: agave-validator --ledger {{ ledger_path }} wait-for-restart-window --max-delinquent-stake {{ max_delinquent_stake }} 18 | environment: 19 | PATH: "{{ env_path }}" 20 | ignore_errors: True 21 | when: wait_for_window and agave is defined and agave 22 | tags: 23 | - validator.manage.restart.wait --------------------------------------------------------------------------------