├── .gitattributes ├── .gitignore ├── .goreleaser.yml ├── CHANGELOG-RUS.md ├── CHANGELOG.md ├── LICENSE ├── NOTICE ├── OPTIONS-RUS.md ├── OPTIONS.md ├── README-RUS.md ├── README.md ├── conf ├── ctrl │ ├── bsd │ │ ├── ctrl.conf │ │ ├── ips-localhost.conf │ │ └── uss-localhost.conf │ ├── ctrl.conf │ ├── ips-localhost.conf │ └── uss-localhost.conf ├── init.d │ ├── bsd │ │ └── ctrl │ └── linux │ │ └── ctrl ├── logrotate.d │ └── ctrl ├── nginx │ ├── localhost-ssl.conf │ └── localhost.conf └── systemd │ └── ctrl.service ├── db.go ├── file.go ├── get.go ├── go.mod ├── go.sum ├── hlp.go ├── images ├── logo.png └── paypal.png ├── log.go ├── main.go ├── post.go ├── rst.go ├── sch.go ├── scripts └── postinstall.sh └── sys.go /.gitattributes: -------------------------------------------------------------------------------- 1 | * -text 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | # This is an example goreleaser.yaml file with some sane defaults. 2 | # Make sure to check the documentation at http://goreleaser.com 3 | before: 4 | hooks: 5 | 6 | env_files: 7 | github_token: /root/projects/keys/main 8 | 9 | builds: 10 | - 11 | binary: ctrl 12 | id: ctrl 13 | ldflags: -s -w 14 | env: 15 | - CGO_ENABLED=0 16 | goos: 17 | - linux 18 | - freebsd 19 | - openbsd 20 | - netbsd 21 | goarch: 22 | - amd64 23 | - arm64 24 | - ppc64 25 | - mips64 26 | goarm: 27 | - 7 28 | 29 | release: 30 | draft: false 31 | 32 | archives: 33 | - 34 | id: "ctrl" 35 | builds: ['ctrl'] 36 | format: tar.gz 37 | name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}-{{.Arch}}" 38 | files: 39 | - LICENSE 40 | - NOTICE 41 | 42 | nfpms: 43 | - 44 | id: "ctrl" 45 | builds: ['ctrl'] 46 | formats: 47 | - deb 48 | - rpm 49 | vendor: "Eltaline " 50 | homepage: "https://elta.ee/" 51 | maintainer: "Andrey Kuvshinov " 52 | description: "Remote cTRL API Server" 53 | license: "Apache 2.0" 54 | file_name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}-{{.Arch}}" 55 | 56 | bindir: /usr/sbin 57 | 58 | dependencies: 59 | - systemd 60 | - logrotate 61 | 62 | empty_folders: 63 | - /etc/ctrl 64 | - /var/log/ctrl 65 | - /var/lib/ctrl 66 | - /usr/share/ctrl 67 | 68 | contents: 69 | - src: "conf/systemd/ctrl.service" 70 | dst: "/lib/systemd/system/ctrl.service" 71 | 72 | - src: "conf/logrotate.d/ctrl" 73 | dst: "/etc/logrotate.d/ctrl" 74 | 75 | - src: "LICENSE" 76 | dst: "/usr/share/ctrl/LICENSE" 77 | 78 | - src: "NOTICE" 79 | dst: "/usr/share/ctrl/NOTICE" 80 | 81 | - src: "conf/ctrl/ctrl.conf" 82 | dst: "/etc/ctrl/ctrl.conf" 83 | type: config 84 | 85 | - src: "conf/ctrl/ips-localhost.conf" 86 | dst: "/etc/ctrl/ips-localhost.conf" 87 | type: config 88 | 89 | - src: "conf/ctrl/uss-localhost.conf" 90 | dst: "/etc/ctrl/uss-localhost.conf" 91 | type: config 92 | 93 | scripts: 94 | postinstall: "scripts/postinstall.sh" 95 | 96 | scoop: 97 | 98 | commit_author: 99 | name: "Andrey Kuvshinov" 100 | email: syslinux@protonmail.com 101 | homepage: "https://elta.ee/" 102 | license: Apache 2.0 103 | 104 | brews: 105 | - 106 | commit_author: 107 | name: "Andrey Kuvshinov" 108 | email: syslinux@protonmail.com 109 | homepage: "https://elta.ee/" 110 | goarm: 7 111 | 112 | snapshot: 113 | name_template: "{{.Version}}-master" 114 | changelog: 115 | sort: asc 116 | filters: 117 | exclude: 118 | - README 119 | - OPTIONS 120 | - Readme 121 | - Options 122 | - Image 123 | - Script 124 | - Docker 125 | - Typo 126 | - "via upload" 127 | - goreleaser 128 | -------------------------------------------------------------------------------- /CHANGELOG-RUS.md: -------------------------------------------------------------------------------- 1 | Версия: 1.1.11 2 | ======= 3 | 4 | - Исправление функций show/del по типу 5 | 6 | Версия: 1.1.10 7 | ======= 8 | 9 | - Стандартное обновление 10 | 11 | Версия: 1.1.9 12 | ======= 13 | 14 | Исправлено в версии 1.1.9: 15 | 16 | - Конвертация переменной 17 | 18 | Версия: 1.1.8 19 | ======== 20 | 21 | Добавлено в версии 1.1.8: 22 | 23 | - Изменение времени задач в очередях 24 | 25 | Версия: 1.1.7 26 | ======== 27 | 28 | Несовместимости: 29 | 30 | - Больше не поддерживается сборка под solaris/darwin из-за остутствия поддержки pgid 31 | 32 | Исправлено в версии 1.1.7: 33 | 34 | - Исправлено зависание рабочей очереди 35 | - Исправлено завсиание go routines 36 | - Исправлена логика завершения процессов 37 | - Исправлено завершение дочерних процессов исполняемы команд 38 | 39 | Версия: 1.1.6 40 | ======== 41 | 42 | Исправлено в версии 1.1.6: 43 | 44 | - Возможно было исправлено зависание рабочей очереди 45 | 46 | Версия: 1.1.5 47 | ======== 48 | 49 | Добавлено в версии 1.1.5 50 | 51 | - Поля ```interr, intcnt``` для перехвата ошибок и снятия/повторения задач на лету 52 | - Поля ```vinterr, vintcnt``` для перехвата ошибок и снятия/повторения задач на лету в конфигурации сервера 53 | - Поле ```lookout``` булевое, включает или отключает реагирование на ошибки из stdout 54 | 55 | Исправлено в версии 1.1.5: 56 | 57 | - Очистка очереди working, при штатном и после падения, перезапуске сервера 58 | - Улучшено расширенное логирование ошибок в app.log 59 | - Небольшие баг фиксы 60 | 61 | Версия: 1.1.4 62 | ======== 63 | 64 | Добавлено в версии 1.1.4 65 | 66 | - Возврат кода ошибки из шелла 67 | 68 | Версия: 1.1.3 69 | ======== 70 | 71 | Добавлено в версии 1.1.3: 72 | 73 | - Поле replace для /task 74 | 75 | Версия: 1.1.2 76 | ======== 77 | 78 | Исправлено в версии 1.1.2: 79 | 80 | - Ошибка mutex в /run 81 | 82 | Версия: 1.1.1 83 | ======== 84 | 85 | Добавлено в версии 1.1.1: 86 | 87 | - Поиск по типу 88 | 89 | Версия: 1.1.0 90 | ======== 91 | 92 | Добавлено в версии 1.1.0: 93 | 94 | - Ручное управление параметрами ```vthreads/vtimeout/vttltime/vinterval/vrepeaterr/vrepeatcnt``` через метод POST 95 | - Настройка лимита потоков под конкретный тип задач 96 | - Парсинг ошибок и автоматическое повторение выполнения задачи 97 | - Обновлена документация 98 | 99 | Исправлено в версии 1.1.0: 100 | 101 | - Незначительные исправления ошибок 102 | 103 | Версия: 1.0.0 104 | ======== 105 | 106 | - Первый релиз 107 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Version: 1.1.11 2 | ======== 3 | 4 | - Fix show/del by type functions 5 | 6 | Version: 1.1.10 7 | ======== 8 | 9 | - Standart update 10 | 11 | Version: 1.1.9 12 | ======== 13 | 14 | Fixed in version 1.1.9: 15 | 16 | - Variable convertion 17 | 18 | Version: 1.1.8 19 | ======== 20 | 21 | Added in version 1.1.8: 22 | 23 | - Change the time of tasks in queues 24 | 25 | Version: 1.1.7 26 | ======== 27 | 28 | Incompatibilities: 29 | 30 | - Building under solaris / darwin is no longer supported due to lack of pgid support 31 | 32 | Fixed in version 1.1.7: 33 | 34 | - Fixed working queue hanging 35 | - Fixed hanging go routines 36 | - Fixed process termination logic 37 | - Fixed termination of child processes of executable commands 38 | 39 | Version: 1.1.6 40 | ======== 41 | 42 | Fixed in version 1.1.6: 43 | 44 | - May be was fix stuck working queue 45 | 46 | Version: 1.1.5 47 | ======== 48 | 49 | Added in version 1.1.5: 50 | 51 | - ```interr, intcnt``` fields for intercept errors and kill/repeat tasks on the fly 52 | - ```vinterr, vintcnt``` fields for intercept errors and kill/repeat tasks on the fly in server configuration 53 | - ```lookout``` boolean, enable or disable reaction on errors from stdout 54 | 55 | Fixed in version 1.1.5: 56 | 57 | - Clearing the working queue, in case of regular and after crash, server restart 58 | - Improved extended error logging in app.log 59 | - Minor bug fixes 60 | 61 | Version: 1.1.4 62 | ======== 63 | 64 | Added in version 1.1.4: 65 | 66 | - Return error status code from shell 67 | 68 | Version: 1.1.3 69 | ======== 70 | 71 | Added in version 1.1.3: 72 | 73 | - Field replace for /task 74 | 75 | Version: 1.1.2 76 | ======== 77 | 78 | Fixed in version 1.1.2: 79 | 80 | - Error mutex in /run 81 | 82 | Version: 1.1.1 83 | ======== 84 | 85 | Added in version 1.1.1: 86 | 87 | - Search by type 88 | 89 | Version: 1.1.0 90 | ======== 91 | 92 | Added in version 1.1.0: 93 | 94 | - Manual control of the parameters ```vthreads/vtimeout/vttltime/vinterval/vrepeaterr/vrepeatcnt``` through the POST method 95 | - Setting the thread limit for a specific type of task 96 | - Parsing errors and automatic task completion 97 | - Updated documentation 98 | 99 | Fixed in version 1.1.0: 100 | 101 | - Minor bug fixes 102 | 103 | Version: 1.0.0 104 | ======== 105 | 106 | - First Release 107 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright © 2020 Andrey Kuvshinov. Contacts: 2 | Copyright © 2020 Eltaline OU. Contacts: 3 | All rights reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | The cTRL project contains unmodified/modified libraries imports too with 18 | separate copyright notices and license terms. Your use of the source code 19 | this libraries is subject to the terms and conditions of licenses these libraries. 20 | -------------------------------------------------------------------------------- /OPTIONS-RUS.md: -------------------------------------------------------------------------------- 1 | Параметры cTRL: 2 | ======== 3 | 4 | Секция [command line] 5 | ------------ 6 | 7 | - -config string 8 | --config=/etc/ctrl/ctrl.conf (по умолчанию "/etc/ctrl/ctrl.conf") 9 | - -debug 10 | --debug - Режим отладки. 11 | - -help 12 | --help - Выводит помощь. 13 | - -version 14 | --version - Выводит версию. 15 | 16 | Секция [global] 17 | ------------ 18 | 19 | bindaddr 20 | - **Описание:** Основной адрес и TCP порт, значение ":9691" все адреса. 21 | - **Умолчание:** "127.0.0.1:9691" 22 | - **Тип:** string 23 | - **Секция:** [global] 24 | 25 | bindaddrssl 26 | - **Описание:** Основной SSL адрес и TCP порт, значение ":9791" все адреса. 27 | - **Умолчание:** "127.0.0.1:9791" 28 | - **Тип:** string 29 | - **Секция:** [global] 30 | 31 | onlyssl 32 | - **Описание:** Глобальное отключение для всех виртуальных хостов обычного HTTP адреса и порта. Не настраивается для виртуального хоста. 33 | - **Умолчание:** false 34 | - **Значения:** true или false 35 | - **Тип:** bool 36 | - **Секция:** [global] 37 | 38 | readtimeout 39 | - **Описание:** Задает таймаут на максимальное время передачи данных от клиента серверу. Для передачи больших файлов следует увеличить. Если cTRL сервер установлен за Nginx или HAProxy, тогда можно не использовать данный таймаут, установив его в 0 (без таймаута). Не настраивается для виртуального хоста. 40 | - **Умолчание:** 60 41 | - **Значения:** 0-86400 42 | - **Тип:** int 43 | - **Секция:** [global] 44 | 45 | readheadertimeout 46 | - **Описание:** Задает таймаут на максимальное время получения заголовков от клиента. Если cTRL сервер установлен за Nginx или HAProxy, тогда можно не использовать данный таймаут, установив его в 0. Если параметр = 0, тогда берется значение параметра readtimeout, если и readtimeout = 0, тогда readheadertimeout таймаут не используется (без таймаута). Не настраивается для виртуального хоста. 47 | - **Умолчание:** 5 48 | - **Значения:** 0-86400 49 | - **Тип:** int 50 | - **Секция:** [global] 51 | 52 | writetimeout 53 | - **Описание:** Задает таймаут на максимальное время передачи данных клиенту. Для передачи больших файлов следует значительно увеличить. Если cTRL сервер установлен за Nginx или HAProxy, тогда можно не использовать данный таймаут установив его в 0 (без таймаута). Не настраивается для виртуального хоста. 54 | - **Умолчание:** 60 55 | - **Значения:** 0-86400 56 | - **Тип:** int 57 | - **Секция:** [global] 58 | 59 | idletimeout 60 | - **Описание:** Задает таймаут на максимальное время жизни keep alive соединений. Если параметр = 0, тогда берется значение параметра readtimeout, если и readtimeout = 0, тогда idletimeout таймаут не используется (без таймаута). Не настраивается для виртуального хоста. 61 | - **Умолчание:** 60 62 | - **Значения:** 0-86400 63 | - **Тип:** int 64 | - **Секция:** [global] 65 | 66 | keepalive 67 | - **Описание:** Глобальное включение или отключение keep alive. Не настраивается для виртуального хоста. 68 | - **Умолчание:** false 69 | - **Значения:** true или false 70 | - **Тип:** bool 71 | - **Секция:** [global] 72 | 73 | realheader 74 | - **Описание:** Заголовок реального IP адреса от реверс прокси. Не настраивается для виртуального хоста. 75 | - **Умолчание:** "X-Real-IP" 76 | - **Тип:** string 77 | - **Секция:** [global] 78 | 79 | charset 80 | - **Описание:** Кодировка на весь сервер. Не настраивается для виртуального хоста. 81 | - **Умолчание:** "UTF-8" 82 | - **Тип:** string 83 | - **Секция:** [global] 84 | 85 | debugmode 86 | - **Описание:** Режим отладки. Не настраивается для виртуального хоста. 87 | - **Умолчание:** false 88 | - **Значения:** true или false 89 | - **Тип:** bool 90 | - **Секция:** [global] 91 | 92 | gcpercent 93 | - **Описание:** Глобально задает параметр сборки мусора (проценты). -1 отключает сборщик мусора. Не настраивается для виртуального хоста. 94 | - **Умолчание:** 25 95 | - **Значения:** -1-100 96 | - **Тип:** int 97 | - **Секция:** [global] 98 | 99 | dbdir 100 | - **Описание:** Директория базы данных задач. Можно безопасно удалить и реинициализировать снова, в этом случае выполненнные задачи тоже удалятся. 101 | - **Умолчание:** "/var/lib/ctrl/db" 102 | - **Тип:** string 103 | - **Секция:** [global] 104 | 105 | schtime = 5 106 | - **Описание:** Интервал чтения полученных задач (секунды). 107 | - **Умолчание:** 5 108 | - **Значения:** 1-3600 109 | - **Тип:** int 110 | - **Секция:** [global] 111 | 112 | pidfile 113 | - **Описание:** Путь к pid файлу. 114 | - **Умолчание:** "/run/ctrl/ctrl.pid" 115 | - **Тип:** string 116 | - **Секция:** [global] 117 | 118 | logdir 119 | - **Описание:** Путь к log директории. 120 | - **Умолчание:** "/var/log/ctrl" 121 | - **Тип:** string 122 | - **Секция:** [global] 123 | 124 | logmode 125 | - **Описание:** Права на log файлы (маска). 126 | - **Умолчание:** 0640 127 | - **Значения:** 0600-0666 128 | - **Тип:** uint32 129 | - **Секция:** [global] 130 | 131 | Секция [server] и подсекции [server.name] 132 | ------------ 133 | 134 | [server.name] 135 | - **Описание:** Основной внутренний идентификатор виртуального хоста. после "." можно использовать любое имя, это не доменное имя, только внутренний идентификатор. 136 | - **Тип:** string 137 | - **Секция:** [server] 138 | 139 | host 140 | - **Описание:** Имя виртуального хоста. Не поддерживается * или _ , для сведения множества виртуальных хостов на 1 виртуальный хост в cTRL сервере используйте Nginx или HAProxy или любой другой реверс прокси сервер с жестко заданным "proxy_set_header Host hostname;" (на примере Nginx), где hostname = host в виртуальном хосте cTRL сервера. 141 | - **Умолчание:** Обязательный параметр 142 | - **Тип:** string 143 | - **Секция:** [server.name] 144 | 145 | sslcrt 146 | - **Описание:** Путь к файлу SSL сертификата для виртуального хоста. 147 | - **Умолчание:** "" 148 | - **Тип:** string 149 | - **Секция:** [server.name] 150 | 151 | sslkey 152 | - **Описание:** Путь к файлу SSL ключа для виртуального хоста. 153 | - **Умолчание:** "" 154 | - **Тип:** string 155 | - **Секция:** [server.name] 156 | 157 | ussallow 158 | - **Описание:** Путь к файлу разрешенных пар login:pass(sha512) для методов GET/POST для виртуального хоста. 159 | - **Умолчание:** "" 160 | - **Тип:** string 161 | - **Секция:** [server.name] 162 | 163 | ipsallow 164 | - **Описание:** Путь к файлу разрешенных IP адресов для методов GET/POST для виртуального хоста. 165 | - **Умолчание:** "" 166 | - **Тип:** string 167 | - **Секция:** [server.name] 168 | 169 | shell 170 | - **Описание:** Задает shell для виртуального хоста. 171 | - **Умолчание:** "/bin/bash" 172 | - **Тип:** string 173 | - **Секция:** [server.name] 174 | 175 | rthreads 176 | - **Описание:** Максимальное количество одновременно запущенных параллельных задач в реальном времени. 177 | - **Умолчание:** 8 178 | - **Значения:** 1-4096 179 | - **Тип:** uint32 180 | - **Секция:** [server.name] 181 | 182 | vthreads 183 | - **Описание:** Максимальное количество одновременно запущенных параллельных задач в режиме очереди. 184 | - **Умолчание:** 8 185 | - **Значения:** 1-4096 186 | - **Тип:** uint32 187 | - **Секция:** [server.name] 188 | 189 | vtimeout 190 | - **Описание:** Задает таймаут по умолчанию для исполняемых задач, если не был ранее задан таймаут в задаче (секунды). 191 | - **Умолчание:** 28800 192 | - **Значения:** 1-2592000 193 | - **Тип:** uint32 194 | - **Секция:** [server.name] 195 | 196 | vttltime 197 | - **Описание:** Задает по умолчанию время жизни по умолчанию для выполненных задач в базе данных (секунды). 198 | - **Умолчание:** 86400 199 | - **Значения:** 1-2592000 200 | - **Тип:** uint32 201 | - **Секция:** [server.name] 202 | 203 | vinterval 204 | - **Описание:** Задает по умолчанию интервал между запуском задач (секунды). 205 | - **Умолчание:** 0 206 | - **Значения:** 0-60 207 | - **Тип:** uint32 208 | - **Секция:** [server.name] 209 | 210 | vrepeaterr 211 | - **Описание:** Задает по умолчанию парсинг ошибок при выполнении задач (после выполнения задачи), можно указать несколько ошибок при которых заработают повторы выполнения задач. 212 | - **Умолчание:** 213 | - **Значения:** 214 | - **Тип:** []string 215 | - **Секция:** [server.name] 216 | 217 | vrepeatcnt 218 | - **Описание:** Задает по умолчанию количество повторных запусков задачи в случае получения одной из ошибок указанных в параметре vrepeaterr. 219 | - **Умолчание:** 0 220 | - **Значения:** 0-1000 221 | - **Тип:** uint32 222 | - **Секция:** [server.name] 223 | 224 | vinterr 225 | - **Описание:** Задает по умолчанию парсинг ошибок при выполнении задач (во время выполнения задачи), можно указать несколько ошибок при которых заработают снятие задач и повторы выполнения задач. 226 | - **Умолчание:** 227 | - **Значения:** 228 | - **Тип:** []string 229 | - **Секция:** [server.name] 230 | 231 | vintcnt 232 | - **Описание:** Задает по умолчанию количество повторных запусков задачи в случае получения одной из ошибок указанных в параметре vinterr. 233 | - **Умолчание:** 0 234 | - **Значения:** 0-1000 235 | - **Тип:** uint32 236 | - **Секция:** [server.name] 237 | -------------------------------------------------------------------------------- /OPTIONS.md: -------------------------------------------------------------------------------- 1 | cTRL Parameters: 2 | ======== 3 | 4 | Section [command line] 5 | ------------ 6 | 7 | - -config string 8 | --config=/etc/ctrl/ctrl.conf (default is "/etc/ctrl/ctrl.conf") 9 | - -debug 10 | --debug - debug mode 11 | - -help 12 | --help - displays help 13 | - -version 14 | --version - print version 15 | 16 | Section [global] 17 | ------------ 18 | 19 | bindaddr 20 | - **Description**: This is the primary address and TCP port. The value is ": 9699" for all addresses. 21 | - **Default:** "127.0.0.1:9699" 22 | - **Type:** string 23 | - **Section:** [global] 24 | 25 | bindaddrssl 26 | - **Description:** This is the primary SSL address and TCP port. The value is ": 9699" for all addresses. 27 | - **Default:** "127.0.0.1:9799" 28 | - **Type:** string 29 | - **Section:** [global] 30 | 31 | onlyssl 32 | - **Description:** This globally disables standart HTTP address and port for all virtual hosts. It is not configured for a virtual host. 33 | - **Default:** false 34 | - **Values:** true or false 35 | - **Type:** bool 36 | - **Section:** [global] 37 | 38 | readtimeout 39 | - **Description:** This sets the timeout for the maximum data transfer time from the client to the server. It should be increased to transfer large files. If the cTRL server is installed behind Nginx or HAProxy, this timeout can be disabled by setting it to 0 (no timeout). It is not configured for a virtual host. 40 | - **Default:** 60 41 | - **Values:** 0-86400 42 | - **Type:** int 43 | - **Section:** [global] 44 | 45 | readheadertimeout 46 | - **Description:** This sets the timeout for the maximum time for receiving headers from the client. If the cTRL server is installed behind Nginx or HAProxy, this timeout can be disabled by setting it to 0. If the parameter = 0, the readtimeout parameter is taken. If the readtimeout = 0, the readheadertimeout timeout is not used (no timeout). It is not configured for a virtual host. 47 | - **Default:** 5 48 | - **Values:** 0-86400 49 | - **Type:** int 50 | - **Section:** [global] 51 | 52 | writetimeout 53 | - **Description:** This sets the timeout for the maximum data transfer time to the client. The transfer of large files should be significantly increased. If the cTRL server is installed behind Nginx or HAProxy, this timeout can be disabled by setting it to 0 (no timeout). It is not configured for a virtual host. 54 | - **Default:** 60 55 | - **Values:** 0-86400 56 | - **Type:** int 57 | - **Section:** [global] 58 | 59 | idletimeout 60 | - **Description:** This sets the timeout for the maximum lifetime of keep alive connections. If the parameter = 0, the readtimeout parameter value is taken. If the readtimeout = 0, the idletimeout timeout is not used (no timeout). It is not configured for a virtual host. 61 | - **Default:** 60 62 | - **Values:** 0-86400 63 | - **Type:** int 64 | - **Section:** [global] 65 | 66 | keepalive 67 | - **Description:** This globally enables or disables keep alive. It is not configured for a virtual host. 68 | - **Default:** false 69 | - **Values:** true or false 70 | - **Type:** bool 71 | - **Section:** [global] 72 | 73 | realheader 74 | - **Description:** This is the real IP address header from the reverse proxy. It is not configured for a virtual host. 75 | - **Default:** "X-Real-IP" 76 | - **Type:** string 77 | - **Section:** [global] 78 | 79 | charset 80 | - **Description:** This is the encoding used for the entire server. It is not configured for a virtual host. 81 | - **Default:** "UTF-8" 82 | - **Type:** string 83 | - **Section:** [global] 84 | 85 | debugmode 86 | - **Description:** This globally enables debug mode. It is not configured for a virtual host. 87 | - **Default:** false 88 | - **Values:** true or false 89 | - **Type:** bool 90 | - **Section:** [global] 91 | 92 | gcpercent 93 | - **Description:** This globally sets the garbage collection parameter (percent). -1 disables the garbage collector. It is not configured for a virtual host. 94 | - **Default:** 25 95 | - **Values:** -1-100 96 | - **Type:** int 97 | - **Section:** [global] 98 | 99 | dbdir 100 | - **Description:** Directory for tasks database. You can safely remove data and reinitialize again, in this case compeleted task too deleted. 101 | - **Default:** "/var/lib/ctrl/db" 102 | - **Type:** string 103 | - **Section:** [global] 104 | 105 | schtime = 5 106 | - **Description:** This is the compaction/defragmentation delay timeout for the updated Bolt archives (days). 107 | - **Default:** 5 108 | - **Values:** 1-3600 109 | - **Type:** int 110 | - **Section:** [global] 111 | 112 | pidfile 113 | - **Description:** This is the PID file path. 114 | - **Default:** "/run/ctrl/ctrl.pid" 115 | - **Type:** string 116 | - **Section:** [global] 117 | 118 | logdir 119 | - **Description:** This is the path to the log directory. 120 | - **Default:** "/var/log/ctrl" 121 | - **Type:** string 122 | - **Section:** [global] 123 | 124 | logmode 125 | - **Description:** This sets the permissions for the log files (mask). 126 | - **Default:** 0640 127 | - **Values:** 0600-0666 128 | - **Type:** uint32 129 | - **Section:** [global] 130 | 131 | Section [server] and subsections [server.name] 132 | ------------ 133 | 134 | [server.name] 135 | - **Description:** This is the primary internal identifier of the virtual host. After "." any name can be used. This is not a domain name, only an internal identifier. 136 | - **Type:** string 137 | - **Section:** [server] 138 | 139 | host 140 | - **Description:** This is the virtual host name. The value * or _ is not supported. To convert multiple virtual hosts to one virtual host in a cTRL server, use Nginx or HAProxy, or any other reverse proxy server with a hard-set "proxy_set_header Host hostname;" (using Nginx as an example) where hostname = host in the cTRL server virtual host. 141 | - **Default:** Required 142 | - **Type:** string 143 | - **Section:** [server.name] 144 | 145 | sslcrt 146 | - **Description:** This is the path to the file with SSL certificate of virtual host. 147 | - **Default:** "" 148 | - **Type:** string 149 | - **Section:** [server.name] 150 | 151 | sslkey 152 | - **Description:** This is the path to the file with SSL key of virtual host. 153 | - **Default:** "" 154 | - **Type:** string 155 | - **Section:** [server.name] 156 | 157 | ussallow 158 | - **Description:** This is the path to the file of login:pass(sha512) pairs for the GET/POST methods for the virtual host. 159 | - **Default:** "" 160 | - **Type:** string 161 | - **Section:** [server.name] 162 | 163 | ipsallow 164 | - **Description:** This is the path to the file of allowed IP addresses for the GET/POST methods for the virtual host. 165 | - **Default:** "" 166 | - **Type:** string 167 | - **Section:** [server.name] 168 | 169 | shell 170 | - **Description:** Sets the shell for the virtual host. 171 | - **Default:** "/bin/bash" 172 | - **Type:** string 173 | - **Section:** [server.name] 174 | 175 | rthreads 176 | - **Description:** Maximum number of concurrently running parallel tasks in real time. 177 | - **Default:** 8 178 | - **Values:** 1-4096 179 | - **Type:** uint32 180 | - **Section:** [server.name] 181 | 182 | vthreads 183 | - **Description:** Maximum number of concurrently running parallel tasks in queue mode. 184 | - **Default:** 8 185 | - **Values:** 1-4096 186 | - **Type:** uint32 187 | - **Section:** [server.name] 188 | 189 | vtimeout 190 | - **Description:** This sets default timeout for running tasks, if timeout is already not set in task (seconds). 191 | - **Default:** 28800 192 | - **Values:** 1-2592000 193 | - **Type:** uint32 194 | - **Section:** [server.name] 195 | 196 | vttltime 197 | - **Description:** This sets default lifetime for completed tasks in the database (seconds). 198 | - **Default:** 86400 199 | - **Values:** 1-2592000 200 | - **Type:** uint32 201 | - **Section:** [server.name] 202 | 203 | vinterval 204 | - **Description:** This sets default interval between starting tasks (seconds). 205 | - **Default:** 0 206 | - **Values:** 0-60 207 | - **Type:** uint32 208 | - **Section:** [server.name] 209 | 210 | vrepeaterr 211 | - **Description:** This sets default parsing of errors when performing tasks(after executed task); you can specify several errors in which repetitions of task execution will work. 212 | - **Default:** 213 | - **Values:** 214 | - **Type:** []string 215 | - **Section:** [server.name] 216 | 217 | vrepeatcnt 218 | - **Description:** This sets default number of task re-starts if one of the errors specified in the vrepeaterr parameter is received. 219 | - **Default:** 0 220 | - **Values:** 0-1000 221 | - **Type:** uint32 222 | - **Section:** [server.name] 223 | 224 | vinterr 225 | - **Description:** This sets default parsing of errors when performing tasks(during executing task); you can specify several errors in which kill task and repetitions of task execution will work. 226 | - **Default:** 227 | - **Values:** 228 | - **Type:** []string 229 | - **Section:** [server.name] 230 | 231 | vintcnt 232 | - **Description:** This sets default number of task re-starts if one of the errors specified in the vinterr parameter is received. 233 | - **Default:** 0 234 | - **Values:** 0-1000 235 | - **Type:** uint32 236 | - **Section:** [server.name] 237 | -------------------------------------------------------------------------------- /README-RUS.md: -------------------------------------------------------------------------------- 1 | cTRL Logo 2 | 3 | cTRL это сервер написанный на языке Go, использующий модифицированную версию NutsDB базы, как бекенд для непрерывной очереди задач и сохранения результата исполнения команд из заданных задач в командных интерпретаторах типа /bin/bash на серверах, где данный сервис будет использоваться. При помощи cTRL можно получать через HTTP протокол задачи с командами для их исполнения на сервере и ограничивать количество одновременно исполняемых задач. 4 | 5 | Текущая стабильная версия: 1.1.11 6 | ======== 7 | 8 | - Changelog 9 | 10 | Возможности 11 | ======== 12 | 13 | - Многопоточность 14 | - Поддержка HTTPS и Auth/IP авторизации 15 | - Поддерживаемые HTTP методы: GET, POST 16 | - Прием задач в очередь для отложенного параллельного исполнения команд 17 | - Параллельное исполнение задач в реальном времени 18 | - Ограничение максимального количества потоков на каждый виртуальный хост 19 | - Поддерживает непрерывную очередь исполнения задач 20 | - Автоматическое одновременное исполнение только одной одинаковой задачи 21 | - Поддерживаемые интерпретаторы: /bin/bash , /bin/sh 22 | - Поддерживаемые форматы: JSON 23 | 24 | Требования 25 | ======== 26 | 27 | - Операционные системы: Linux, BSD, Solaris и OSX 28 | - Архитектуры: AMD64, ARM64, PPC64 и MIPS64, проверялась только AMD64 29 | 30 | Реальное применение 31 | ======== 32 | 33 | Мы используем данный сервер для исполнения отложенных задач по параллельной конвертации видео с помощью ffmpeg и ограничением процессов на нескольких серверах с Nvidia GPU, но сервер может исполнять любые команды через shell интерпретаторы. 34 | 35 | Документация 36 | ======== 37 | 38 | Установка 39 | -------- 40 | 41 | Установка пакетов/бинарников 42 | -------- 43 | 44 | - Скачать 45 | 46 | ``` 47 | systemctl enable ctrl && systemctl start ctrl 48 | ``` 49 | 50 | Настройка и использование cTRL сервера 51 | -------- 52 | 53 | Учетные данные по умолчанию: ```admin:swordfish``` 54 | 55 | Генерация пароля: ```echo -n "mypassword" | sha512sum``` 56 | 57 | В большинстве случаев достаточно воспользоваться конфигурационным файлом по умолчанию. Полное описание всех параметров продукта доступно здесь: Опции 58 | 59 | В данном руководстве используются идентификаторы типа UUID. Но клиент может устанавливать идентификаторы задач в произвольном формате. 60 | 61 | Основные методы 62 | -------- 63 | 64 | Запуск задачи или списка задач в реальном времени с ожиданием исполнения 65 | 66 | ```bash 67 | curl -X POST -H "Auth: login:pass" "Content-Type: application/json" -d @task.json http://localhost/run 68 | ``` 69 | 70 | Постановка задачи или списка задач в очередь 71 | 72 | ```bash 73 | curl -X POST -H "Auth: login:pass" -H "Content-Type: application/json" -d @task.json http://localhost/task 74 | ``` 75 | 76 | Получение задачи из очереди ожидания 77 | 78 | ```bash 79 | curl -H "Auth: login:pass" "http://localhost/show?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=received" 80 | ``` 81 | 82 | Получение всех задач из очереди ожидания 83 | 84 | ```bash 85 | curl -H "Auth: login:pass" "http://localhost/show?queue=received" 86 | ``` 87 | 88 | Получение задачи из рабочей очереди 89 | 90 | ```bash 91 | curl -H "Auth: login:pass" "http://localhost/show?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=working" 92 | ``` 93 | 94 | Получение всех задач из рабочей очереди 95 | 96 | ```bash 97 | curl -H "Auth: login:pass" "http://localhost/show?queue=working" 98 | ``` 99 | 100 | Получение задачи из списка завершенных задач 101 | 102 | ```bash 103 | curl -H "Auth: login:pass" "http://localhost/show?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=completed" 104 | ``` 105 | 106 | Получение всех задач из списка завершенных задач 107 | 108 | ```bash 109 | curl -H "Auth: login:pass" "http://localhost/show?queue=completed" 110 | ``` 111 | 112 | Удаление задачи из очереди ожидания 113 | 114 | ```bash 115 | curl -H "Auth: login:pass" "http://localhost/del?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=received" 116 | ``` 117 | 118 | Удаление всех задач из очереди ожидания 119 | 120 | ```bash 121 | curl -H "Auth: login:pass" "http://localhost/del?queue=received" 122 | ``` 123 | 124 | Удаление задачи из рабочей очереди 125 | 126 | ```bash 127 | curl -H "Auth: login:pass" "http://localhost/del?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=working" 128 | ``` 129 | 130 | Удаление всех задач из рабочей очереди 131 | 132 | ```bash 133 | curl -H "Auth: login:pass" "http://localhost/del?queue=working" 134 | ``` 135 | 136 | Удаление задачи из списка завершенных задач 137 | 138 | ```bash 139 | curl -H "Auth: login:pass" "http://localhost/del?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=completed" 140 | ``` 141 | 142 | Удаление всех задач из списка завершенных задач 143 | 144 | ```bash 145 | curl -H "Auth: login:pass" "http://localhost/del?queue=completed" 146 | ``` 147 | 148 | Формат ввода 149 | -------- 150 | 151 | Описание полей 152 | -------- 153 | 154 | - key - произвольный уникальный идентификатор (не допускается использовать двоеточие ":") 155 | - type - произвольный тип задачи (не допускается использовать двоеточие ":") 156 | - path - путь для смены директории перед запуском команды 157 | - lock - произвольная метка блокировки 158 | - command - команда 159 | - threads - количество потоков для определенного типа задач 160 | - timeout - таймаут 161 | - ttltime - время жизни задачи в completed очереди 162 | - interval - интервал между запуском задач 163 | - repeaterr - перечисление ошибок, которые требуют повторного выполнения задачи 164 | - repeatcnt - количество повторных запусков задачи в случае совпадения с любой ошибкой из параметра ```repeaterr``` 165 | - interr - перечисление ошибок, которые требуют снятия запущенной задачи 166 | - intcnt - количество повторных запусков задачи в случае совпадения с любой ошибкой из параметра ```interr``` 167 | - lookout - включает или отключает реагирование на ошибки из stdout 168 | - replace - перезапись одинаковой задачи с одинаковым ключем и одинаковым типом в очереди received 169 | 170 | Поля ```threads/ttltime/interval/repeaterr/repeatcnt/interr/intcnt/lookout/replace``` актуальны только для задач установленных в очередь 171 | 172 | Примеры установки задач 173 | -------- 174 | 175 | Пример для одной задачи через /run: 176 | 177 | ```json 178 | [ 179 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","timeout":15} 180 | ] 181 | ``` 182 | 183 | Пример для списка задач через /run: 184 | 185 | ```json 186 | [ 187 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","timeout":15}, 188 | {"key":"4964deca-46ff-413f-8a92-e5baefd328e7","type":"mytype","path":"/","lock":"mylock2","command":"echo \"great\" && logger \"great\" && sleep 30","timeout":15}, 189 | {"key":"3fdf744d-36f1-499d-bd39-90a004ee39f6","type":"mytype","path":"/","lock":"mylock3","command":"echo \"world\" && logger \"world\" && sleep 15","timeout":15} 190 | ] 191 | ``` 192 | 193 | Пример для одной задачи через /task: 194 | 195 | ```json 196 | [ 197 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false} 198 | ] 199 | ``` 200 | 201 | Пример для списка задач через /task: 202 | 203 | ```json 204 | [ 205 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false}, 206 | {"key":"4964deca-46ff-413f-8a92-e5baefd328e7","type":"mytype","path":"/","lock":"mylock2","command":"echo \"great\" && logger \"great\" && sleep 30","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false}, 207 | {"key":"3fdf744d-36f1-499d-bd39-90a004ee39f6","type":"mytype","path":"/","lock":"mylock3","command":"echo \"world\" && logger \"world\" && sleep 15","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false} 208 | ] 209 | ``` 210 | 211 | Формат вывода 212 | -------- 213 | 214 | Описание полей 215 | -------- 216 | 217 | - key - произвольный уникальный идентификатор 218 | - time - время приема задачи 219 | - type - произвольный тип задачи 220 | - path - путь для смены директории перед запуском команды 221 | - lock - произвольная метка блокировки 222 | - command - команда 223 | - threads - количество потоков для определенного типа задач 224 | - timeout - таймаут 225 | - ttltime - время жизни задачи в completed очереди 226 | - interval - интервал между запуском задач 227 | - repeaterr - перечисление ошибок, которые требуют повторного выполнения задачи 228 | - repeatcnt - количество повторных запусков задачи в случае совпадения с любой ошибкой из параметра ```repeaterr``` 229 | - interr - перечисление ошибок, которые требуют снятия запущенной задачи 230 | - intcnt - количество повторных запусков задачи в случае совпадения с любой ошибкой из параметра ```interr``` 231 | - lookout - включает или отключает реагирование на ошибки из stdout 232 | - replace - перезапись одинаковой задачи с одинаковым ключем и одинаковым типом в очереди received 233 | - stdcode - не используется на данный момент 234 | - stdout - стандартный вывод 235 | - errcode - код ошибки 236 | - stderr - стандартный вывод ошибки 237 | - runtime - время выполнения в миллисекундах 238 | - delcode - код ошибки при удалении задачи 239 | - delerr - сообщение с ошибкой при удалении задачи 240 | 241 | Примеры вывода 242 | -------- 243 | 244 | Вывод завершенных задач: 245 | 246 | ```json 247 | [ 248 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","time":1589737139,"type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"hello\n","errcode":0,"stderr":"","runtime":10010.669069}, 249 | {"key":"4964deca-46ff-413f-8a92-e5baefd328e7","time":1589737139,"type":"mytype","path":"/","lock":"mylock2","command":"echo \"great\" && logger \"great\" && sleep 30","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"great\n","errcode":124,"stderr":"signal: killed","runtime":15006.034832}, 250 | {"key":"3fdf744d-36f1-499d-bd39-90a004ee39f6","time":1589737139,"type":"mytype","path":"/","lock":"mylock3","command":"echo \"world\" && logger \"world\" && sleep 15","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"world\n","errcode":0,"stderr":"","runtime":15019.839685} 251 | ] 252 | ``` 253 | 254 | Вывод удаленных задач: 255 | 256 | ```json 257 | [ 258 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","time":1589737139,"type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"hello\n","errcode":0,"stderr":"","runtime":10010.669069,"delcode":0,"delerr":""}, 259 | {"key":"4964deca-46ff-413f-8a92-e5baefd328e7","time":1589737139,"type":"mytype","path":"/","lock":"mylock2","command":"echo \"great\" && logger \"great\" && sleep 30","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"great\n","errcode":124,"stderr":"signal: killed","runtime":15006.034832,"delcode":0,"delerr":""}, 260 | {"key":"3fdf744d-36f1-499d-bd39-90a004ee39f6","time":1589737139,"type":"mytype","path":"/","lock":"mylock3","command":"echo \"world\" && logger \"world\" && sleep 15","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"world\n","errcode":0,"stderr":"","runtime":15019.839685,"delcode":0,"delerr":""} 261 | ] 262 | ``` 263 | 264 | Примечания и Q&A 265 | -------- 266 | 267 | - В ключе и типе не допускается использовать двоеточие ":" 268 | - Ограничение одновременно запущенных задач из очереди на каждый виртуальный хост регулируется посредством параметра vthreads в конфигурационном файле сервера или может быть переопределено через метод POST для каждого типа задачи 269 | - Ограничение одновременно запущенных задач в реальном времени на каждый виртуальный хост регулируется посредством параметра rthreads в конфигурационном файле сервера 270 | - Поле key, если данный идентификатор будет одинаковым для двух и более разных задач, в таком случае при выводе информации из очереди вы будете получать по данному идентификатору информацию сразу по нескольким задачам, это может быть полезно для группировки задач, но из очереди ожидания они будут выполняться в произвольном порядке 271 | - Поля type и lock, если они будут назначены двум и более разным задачам абсолютно одинаковыми, в таком случае сервер будет выполнять эти задачи из очереди ожидания в произвольном порядке, но только поочереди 272 | - Поля type и lock, установленные в реальном времени, не имеют значения, но они обязательны, все задачи будут выполняться по возможности в параллельном режиме. 273 | - Для последовательного выполнения списка конкретных связанных друг с другом команд через очередь ожидания, устанавливайте данные команды в одну задачу, разделенные && или же напишите shell скрипт 274 | - Задачи выполняемые в реальном времени исполняются параллельно. 275 | 276 | Ошибки 277 | -------- 278 | 279 | errcode 280 | -------- 281 | 282 | - 0 (нет ошибки) 283 | - 1 (любая ошибка) 284 | - 124 (таймаут) 285 | - 137 (переполнение памяти) 286 | - 255 (не удалось запустить команду) 287 | 288 | delcode 289 | -------- 290 | 291 | - 0 (нет ошибки) 292 | - 1 (любая ошибка) 293 | 294 | ТуДу 295 | ======== 296 | 297 | - Любые предложения 298 | 299 | Параметры 300 | ======== 301 | 302 | Полное описание всех параметров продукта доступно здесь: Опции 303 | 304 | HTTP Core 305 | ======== 306 | 307 | Использует Iris в качестве http ядра 308 | 309 | Гарантии 310 | ======== 311 | 312 | Никакие гарантии на данное программное обеспечение не предоставляются. Просьба сначала тестировать. 313 | 314 | Контакты 315 | ======== 316 | 317 | - Сайт компании: Eltaline 318 | 319 | ``` 320 | Copyright © 2020 Andrey Kuvshinov. Contacts: 321 | Copyright © 2020 Eltaline OU. Contacts: 322 | All rights reserved. 323 | ``` 324 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | cTRL Logo 2 | 3 | Документация на русском: https://github.com/eltaline/ctrl/blob/master/README-RUS.md 4 | 5 | cTRL is a server written in Go language that uses a modified version of the NutsDB database, as a backend for a continuous queue of tasks and saving the result of executing commands from given tasks in command interpreters like /bin/bash on servers where this service will be used. Using cTRL, you can receive tasks via the HTTP protocol with commands for executing them on the server and limit the number of simultaneously executed tasks. 6 | 7 | Current stable version: 1.1.11 8 | ======== 9 | 10 | - Changelog 11 | 12 | Features 13 | ======== 14 | 15 | - Multi threading 16 | - Supports HTTPS and Auth/IP authorization 17 | - Supported HTTP methods: GET, POST 18 | - Parallel execution of tasks in realtime 19 | - Receiving tasks in the queue for deferred parallel execution of commands 20 | - Limiting the maximum number of threads per virtual host 21 | - Supports continuous task execution queue 22 | - Automatic simultaneous execution of only one identical task 23 | - Supported interpreters: /bin/bash, /bin/sh 24 | - Supported formats: JSON 25 | 26 | Requirements 27 | ======== 28 | 29 | - Operating Systems: Linux, BSD, Solaris, OSX 30 | - Architectures: amd64, arm64, ppc64 and mips64, with only amd64 tested 31 | 32 | Real application 33 | ======== 34 | 35 | We use this server to perform delayed tasks for parallel video conversion using ffmpeg with limiting processes on several video servers with Nvidia GPU, but the server can execute any commands through the shell interpreters. This is a first release. 36 | 37 | Documentation 38 | ======== 39 | 40 | Installation 41 | -------- 42 | 43 | Install packages or binaries 44 | -------- 45 | 46 | - Download 47 | 48 | ``` 49 | systemctl enable ctrl && systemctl start ctrl 50 | ``` 51 | 52 | Configuring and using cTRL server 53 | -------- 54 | 55 | Default credentials: ```admin:swordfish``` 56 | 57 | Password generation: ```echo -n "mypassword" | sha512sum``` 58 | 59 | In most cases it is enough to use the default configuration file. A full description of all product parameters is available here: Options 60 | 61 | This guide uses UUIDs. But the client can set task identifiers in any format. 62 | 63 | General methods 64 | -------- 65 | 66 | Run a task or task list in realtime with pending execution 67 | 68 | ```bash 69 | curl -X POST -H "Auth: login:pass" -H "Content-Type: application/json" -d @task.json http://localhost/run 70 | ``` 71 | 72 | Queuing a task or task list 73 | 74 | ```bash 75 | curl -X POST -H "Auth: login:pass" -H "Content-Type: application/json" -d @task.json http://localhost/task 76 | ``` 77 | 78 | Getting a task from the received queue 79 | 80 | ```bash 81 | curl -H "Auth: login:pass" "http://localhost/show?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=received" 82 | ``` 83 | 84 | Getting all tasks from the received queue 85 | 86 | ```bash 87 | curl -H "Auth: login:pass" "http://localhost/show?queue=received" 88 | ``` 89 | 90 | Getting a task from the working queue 91 | 92 | ```bash 93 | curl -H "Auth: login:pass" "http://localhost/show?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=working" 94 | ``` 95 | 96 | Getting all tasks from the working queue 97 | 98 | ```bash 99 | curl -H "Auth: login:pass" "http://localhost/show?queue=working" 100 | ``` 101 | 102 | Getting a task from the list of completed tasks 103 | 104 | ```bash 105 | curl -H "Auth: login:pass" "http://localhost/show?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=completed" 106 | ``` 107 | 108 | Getting all tasks from the list of completed tasks 109 | 110 | ```bash 111 | curl -H "Auth: login:pass" "http://localhost/show?queue=completed" 112 | ``` 113 | 114 | Deleting a task from the received queue 115 | 116 | ```bash 117 | curl -H "Auth: login:pass" "http://localhost/del?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=received" 118 | ``` 119 | 120 | Deleting all tasks from the received queue 121 | 122 | ```bash 123 | curl -H "Auth: login:pass" "http://localhost/del?queue=received" 124 | ``` 125 | 126 | Deleting a task from the working queue 127 | 128 | ```bash 129 | curl -H "Auth: login:pass" "http://localhost/del?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=working" 130 | ``` 131 | 132 | Deleting all tasks from the working queue 133 | 134 | ```bash 135 | curl -H "Auth: login:pass" "http://localhost/del?queue=working" 136 | ``` 137 | 138 | Deleting a task from the list of completed tasks 139 | 140 | ```bash 141 | curl -H "Auth: login:pass" "http://localhost/del?key=777a0d24-289e-4615-a439-0bd4efab6103&type=mytype&queue=completed" 142 | ``` 143 | 144 | Deleting all tasks from the list of completed tasks 145 | 146 | ```bash 147 | curl -H "Auth: login:pass" "http://localhost/del?queue=completed" 148 | ``` 149 | 150 | Input format 151 | -------- 152 | 153 | Description of fields 154 | -------- 155 | 156 | - key - an arbitrary unique identifier (colon is not allowed ":") 157 | - type - arbitrary type of task (colon is not allowed ":") 158 | - path - path to change the directory before running the command 159 | - lock - arbitrary lock label 160 | - command - command 161 | - threads - the number of threads for a particular type of task 162 | - timeout - timeout 163 | - ttltime - task lifetime in completed queue 164 | - interval - the interval between starting tasks 165 | - repeaterr - enumeration of errors that require the repeated execution of a task 166 | - repeatcnt - the number of task reruns in case of any error from parameter ```repeaterr``` 167 | - interr - enumeration of errors that require kill running task 168 | - intcnt - the number of task reruns in case of any error from parameter ```interr``` 169 | - lookout - enable or disable reaction on errors from stdout 170 | - replace - overwrite the same task with the same key and the same type in the received queue 171 | 172 | Fields ```threads/ttltime/interval/repeaterr/repeatcnt/interr/intcnt/lookout/replace``` are relevant only for tasks queued 173 | 174 | Examples of setting tasks 175 | -------- 176 | 177 | Example for one task through /run: 178 | 179 | ```json 180 | [ 181 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","timeout":15} 182 | ] 183 | ``` 184 | 185 | Example for a task list through /run: 186 | 187 | ```json 188 | [ 189 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","timeout":15}, 190 | {"key":"4964deca-46ff-413f-8a92-e5baefd328e7","type":"mytype","path":"/","lock":"mylock2","command":"echo \"great\" && logger \"great\" && sleep 30","timeout":15}, 191 | {"key":"3fdf744d-36f1-499d-bd39-90a004ee39f6","type":"mytype","path":"/","lock":"mylock3","command":"echo \"world\" && logger \"world\" && sleep 15","timeout":15} 192 | ] 193 | ``` 194 | 195 | Example for one task through /task: 196 | 197 | ```json 198 | [ 199 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false} 200 | ] 201 | ``` 202 | 203 | Example for a task list through /task: 204 | 205 | ```json 206 | [ 207 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false}, 208 | {"key":"4964deca-46ff-413f-8a92-e5baefd328e7","type":"mytype","path":"/","lock":"mylock2","command":"echo \"great\" && logger \"great\" && sleep 30","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false}, 209 | {"key":"3fdf744d-36f1-499d-bd39-90a004ee39f6","type":"mytype","path":"/","lock":"mylock3","command":"echo \"world\" && logger \"world\" && sleep 15","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false} 210 | ] 211 | ``` 212 | 213 | Output format 214 | -------- 215 | 216 | Description of fields 217 | -------- 218 | 219 | - key - an arbitrary unique identifier 220 | - time - task receive time 221 | - type - arbitrary type of task 222 | - path - path to change the directory before running the command 223 | - lock - arbitrary lock label 224 | - command - command 225 | - threads - the number of threads for a particular type of task 226 | - timeout - timeout 227 | - ttltime - task lifetime in completed queue 228 | - interval - the interval between starting tasks 229 | - repeaterr - enumeration of errors that require the repeated execution of a task 230 | - repeatcnt - the number of task reruns in case of any error from parameter ```repeaterr``` 231 | - interr - enumeration of errors that require kill running task 232 | - intcnt - the number of task reruns in case of any error from parameter ```interr``` 233 | - lookout - enable or disable reaction on errors from stdout 234 | - replace - overwrite the same task with the same key and the same type in the received queue 235 | - stdcode - not used at the moment 236 | - stdout - standard output 237 | - errcode - error code 238 | - stderr - standard error output 239 | - runtime - runtime in milliseconds 240 | - delcode - error code when deleting a task 241 | - delerr - error message when deleting a task 242 | 243 | Output example 244 | -------- 245 | 246 | Output of completed tasks: 247 | 248 | ```json 249 | [ 250 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","time":1589737139,"type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"hello\n","errcode":0,"stderr":"","runtime":10010.669069}, 251 | {"key":"4964deca-46ff-413f-8a92-e5baefd328e7","time":1589737139,"type":"mytype","path":"/","lock":"mylock2","command":"echo \"great\" && logger \"great\" && sleep 30","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"great\n","errcode":124,"stderr":"signal: killed","runtime":15006.034832}, 252 | {"key":"3fdf744d-36f1-499d-bd39-90a004ee39f6","time":1589737139,"type":"mytype","path":"/","lock":"mylock3","command":"echo \"world\" && logger \"world\" && sleep 15","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"world\n","errcode":0,"stderr":"","runtime":15019.839685} 253 | ] 254 | ``` 255 | 256 | Output of deleted tasks: 257 | 258 | ```json 259 | [ 260 | {"key":"777a0d24-289e-4615-a439-0bd4efab6103","time":1589737139,"type":"mytype","path":"/","lock":"mylock1","command":"echo \"hello\" && logger \"hello\" && sleep 5","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"hello\n","errcode":0,"stderr":"","runtime":10010.669069,"delcode":0,"delerr":""}, 261 | {"key":"4964deca-46ff-413f-8a92-e5baefd328e7","time":1589737139,"type":"mytype","path":"/","lock":"mylock2","command":"echo \"great\" && logger \"great\" && sleep 30","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"great\n","errcode":124,"stderr":"signal: killed","runtime":15006.034832,"delcode":0,"delerr":""}, 262 | {"key":"3fdf744d-36f1-499d-bd39-90a004ee39f6","time":1589737139,"type":"mytype","path":"/","lock":"mylock3","command":"echo \"world\" && logger \"world\" && sleep 15","threads":4,"timeout":15,"ttltime":3600,"interval":1,"repeaterr":["CUDA_ERROR_OUT_OF_MEMORY","OtherError"],"repeatcnt":3,"interr":["Generic error in an external library","OtherError"],"intcnt":1,"lookout":false,"replace":false,"stdcode":0,"stdout":"world\n","errcode":0,"stderr":"","runtime":15019.839685,"delcode":0,"delerr":""} 263 | ] 264 | ``` 265 | 266 | Notes and Q&A 267 | -------- 268 | 269 | - In key and type, colon is not allowed ":" 270 | - The limitation of simultaneously running tasks from queue to each virtual host is regulated by the ```vthreads``` parameter in the server configuration file or can be overriden through POST method for each type of task 271 | - The limitation of simultaneously running tasks in realtime to each virtual host is regulated by the ```rthreads``` parameter in the server configuration file 272 | - The key field, if this identifier is the same for two or more different tasks, in this case, when outputting information from the queue, you will receive information on this identifier for several tasks at once, this can be useful for grouping tasks, but they will be from the waiting queue run randomly 273 | - The type and lock fields, if they are assigned to two or more different tasks, are absolutely identical, in which case the server will perform these tasks from the wait queue in an arbitrary order, but only in turn 274 | - The type and lock fields set in real time do not matter, but they are required, all tasks will be performed in parallel mode whenever possible. 275 | - To sequentially execute a list of specific commands related to each other through a waiting queue, install these commands in one task, separated by && or write a shell script 276 | - Tasks performed in real time are executed parallel. 277 | 278 | Error Codes 279 | -------- 280 | 281 | errcode 282 | -------- 283 | 284 | - 0 (no error) 285 | - 1 (any error) 286 | - 124 (timeout) 287 | - 137 (out of memory) 288 | - 255 (failed to start command) 289 | 290 | delcode 291 | -------- 292 | 293 | - 0 (no error) 294 | - 1 (any error) 295 | 296 | Todo 297 | ======== 298 | 299 | - Any suggestions 300 | 301 | Parameters 302 | ======== 303 | 304 | A full description of all product parameters is available here: Options 305 | 306 | HTTP Core 307 | ======== 308 | 309 | Uses Iris as server http core 310 | 311 | Guarantees 312 | ======== 313 | 314 | No warranty is provided for this software. Please test first 315 | 316 | Contacts 317 | ======== 318 | 319 | - Company website: Eltaline 320 | 321 | ``` 322 | Copyright © 2020 Andrey Kuvshinov. Contacts: 323 | Copyright © 2020 Eltaline OU. Contacts: 324 | All rights reserved. 325 | ``` 326 | -------------------------------------------------------------------------------- /conf/ctrl/bsd/ctrl.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | 3 | bindaddr = "127.0.0.1:9691" 4 | readtimeout = 900 5 | readheadertimeout = 5 6 | writetimeout = 900 7 | idletimeout = 900 8 | keepalive = false 9 | 10 | realheader = "X-Real-IP" 11 | charset = "UTF-8" 12 | 13 | debugmode = false 14 | 15 | gcpercent = 25 16 | 17 | dbdir = "/usr/local/ctrl/lib/ctrl/db" 18 | 19 | schtime = 5 20 | 21 | logdir = "/usr/local/ctrl/log/ctrl" 22 | logmode = 0640 23 | 24 | pidfile = "/usr/local/ctrl/lib/ctrl.pid" 25 | 26 | [server] 27 | 28 | [server.loc] 29 | host = "localhost" 30 | sslcrt = "" 31 | sslkey = "" 32 | ussallow = "/usr/local/ctrl/etc/uss-localhost.conf" 33 | ipsallow = "/usr/local/ctrl/etc/ips-localhost.conf" 34 | shell = "/bin/bash" 35 | rthreads = 64 36 | vthreads = 8 37 | vtimeout = 28800 38 | vttltime = 86400 39 | vinterval = 0 40 | vrepeaterr = ["CUDA_ERROR_OUT_OF_MEMORY","CUDA_ERROR_UNKNOWN","OtherError"] 41 | vrepeatcnt = 10 42 | vinterr = ["Generic error in an external library","OtherError"] 43 | vintcnt = 10 44 | 45 | [end] -------------------------------------------------------------------------------- /conf/ctrl/bsd/ips-localhost.conf: -------------------------------------------------------------------------------- 1 | 127.0.0.0/8 2 | -------------------------------------------------------------------------------- /conf/ctrl/bsd/uss-localhost.conf: -------------------------------------------------------------------------------- 1 | admin:9719a6439375c9115e01dceda86e210e5f2d78a6cf3f4872997746832c4c0f58c5ae0923fabe5acfb923dfc94a117a7d444e453622912dfa193fc6636581f159 2 | -------------------------------------------------------------------------------- /conf/ctrl/ctrl.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | 3 | bindaddr = "127.0.0.1:9691" 4 | readtimeout = 900 5 | readheadertimeout = 5 6 | writetimeout = 900 7 | idletimeout = 900 8 | keepalive = false 9 | 10 | realheader = "X-Real-IP" 11 | charset = "UTF-8" 12 | 13 | debugmode = false 14 | 15 | gcpercent = 25 16 | 17 | dbdir = "/var/lib/ctrl/db" 18 | 19 | schtime = 5 20 | 21 | logdir = "/var/log/ctrl" 22 | logmode = 0640 23 | 24 | pidfile = "/run/ctrl/ctrl.pid" 25 | 26 | [server] 27 | 28 | [server.loc] 29 | host = "localhost" 30 | sslcrt = "" 31 | sslkey = "" 32 | ussallow = "/etc/ctrl/uss-localhost.conf" 33 | ipsallow = "/etc/ctrl/ips-localhost.conf" 34 | shell = "/bin/bash" 35 | rthreads = 64 36 | vthreads = 8 37 | vtimeout = 28800 38 | vttltime = 86400 39 | vinterval = 0 40 | vrepeaterr = ["CUDA_ERROR_OUT_OF_MEMORY","CUDA_ERROR_UNKNOWN","OtherError"] 41 | vrepeatcnt = 10 42 | vinterr = ["Generic error in an external library","OtherError"] 43 | vintcnt = 10 44 | 45 | [end] -------------------------------------------------------------------------------- /conf/ctrl/ips-localhost.conf: -------------------------------------------------------------------------------- 1 | 127.0.0.0/8 2 | -------------------------------------------------------------------------------- /conf/ctrl/uss-localhost.conf: -------------------------------------------------------------------------------- 1 | admin:9719a6439375c9115e01dceda86e210e5f2d78a6cf3f4872997746832c4c0f58c5ae0923fabe5acfb923dfc94a117a7d444e453622912dfa193fc6636581f159 2 | -------------------------------------------------------------------------------- /conf/init.d/bsd/ctrl: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # PROVIDE: ctrl 3 | # REQUIRE: networking syslog 4 | 5 | . /etc/rc.subr 6 | 7 | name="ctrl" 8 | rcvar="ctrl_enable" 9 | 10 | command="/usr/local/ctrl/sbin/ctrl --config=/usr/local/ctrl/etc/ctrl.conf" 11 | 12 | ctrl_user="ctrl" 13 | 14 | ctrl_cnfdir="/usr/local/ctrl/etc" 15 | ctrl_libdir="/usr/local/ctrl/lib" 16 | ctrl_logdir="/usr/local/ctrl/log" 17 | 18 | pidfile="${ctrl_libdir}/${name}.pid" 19 | 20 | start_precmd="ctrl_precmd" 21 | start_cmd="/usr/sbin/daemon -f -u $ctrl_user $command" 22 | gracefulstop_cmd="ctrl_gracefulstop" 23 | 24 | load_rc_config $name 25 | : ${ctrl_enable:=no} 26 | 27 | ctrl_check() 28 | { 29 | 30 | if [ -z `getent group ctrl` ]; then 31 | pw groupadd ctrl 32 | fi 33 | 34 | if [ -z `getent passwd ctrl` ]; then 35 | pw useradd ctrl -g ctrl 36 | fi 37 | 38 | if [ ! -d ${ctrl_cnfdir} ] ; then 39 | install -d -o ctrl -g ctrl -m 755 ${ctrl_cnfdir} 40 | fi 41 | 42 | if [ ! -d ${ctrl_libdir} ] ; then 43 | install -d -o ctrl -g ctrl -m 755 ${ctrl_libdir} 44 | fi 45 | 46 | if [ ! -d ${ctrl_logdir} ] ; then 47 | install -d -o ctrl -g ctrl -m 755 ${ctrl_logdir} 48 | fi 49 | 50 | } 51 | 52 | ctrl_precmd() 53 | { 54 | 55 | ctrl_check 56 | 57 | } 58 | 59 | ctrl_gracefulstop() 60 | { 61 | echo "Performing a graceful stop:" 62 | sig_stop="TERM" 63 | run_rc_command ${rc_prefix}stop || return 1 64 | } 65 | 66 | run_rc_command "$1" 67 | -------------------------------------------------------------------------------- /conf/init.d/linux/ctrl: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ### BEGIN INIT INFO 3 | # Provides: ctrl 4 | # Required-Start: $syslog $local_fs $remote_fs $network 5 | # Required-Stop: $syslog $local_fs $remote_fs 6 | # Default-Start: 2 3 4 5 7 | # Default-Stop: 0 1 6 8 | # Short-Description: cTRL Service 9 | ### END INIT INFO 10 | 11 | . /lib/lsb/init-functions 12 | DAEMON=ctrl 13 | USER="ctrl" 14 | GROUP="ctrl" 15 | DBDIR="/var/lib/ctrl" 16 | LOGDIR="/var/log/ctrl" 17 | RUNDIR="/run/ctrl" 18 | PIDFILE=/run/ctrl/ctrl.pid 19 | DESC="cTRL Service" 20 | start() { 21 | 22 | if [ -z `getent group ctrl` ]; then 23 | groupadd ctrl 24 | fi 25 | 26 | if [ -z `getent passwd ctrl` ]; then 27 | useradd ctrl -g ctrl 28 | fi 29 | 30 | test -d $DBDIR || install --mode=755 --owner=ctrl --group=ctrl --directory $DBDIR 31 | test -d $LOGDIR || install --mode=755 --owner=ctrl --group=ctrl --directory $LOGDIR 32 | test -d $RUNDIR || install --mode=755 --owner=ctrl --group=ctrl --directory $RUNDIR 33 | 34 | log_daemon_msg "Starting $DESC\n" 35 | start-stop-daemon -q -b -u $USER -g $GROUP -p $PIDFILE --start --exec /usr/sbin/ctrl 36 | if [ $? -ne 0 ]; then 37 | log_failure_msg "Failed" 38 | exit 1 39 | fi 40 | if [ $? -eq 0 ]; then 41 | log_success_msg "cTRL Started" 42 | fi 43 | exit 0 44 | 45 | } 46 | 47 | stop() { 48 | 49 | log_daemon_msg "Stopping $DESC\n" 50 | start-stop-daemon -q -p $PIDFILE --stop 51 | if [ $? -ne 0 ]; then 52 | log_failure_msg "Failed" 53 | exit 1 54 | fi 55 | if [ $? -eq 0 ]; then 56 | log_success_msg "cTRL Stopped" 57 | fi 58 | 59 | } 60 | 61 | force_reload() { 62 | 63 | stop 64 | start 65 | 66 | } 67 | 68 | case "$1" in 69 | 70 | start) 71 | start 72 | ;; 73 | stop) 74 | stop 75 | ;; 76 | force-reload) 77 | force_reload 78 | ;; 79 | restart) 80 | stop 81 | start 82 | ;; 83 | 84 | *) 85 | echo "$Usage: $DAEMON {start|stop|force-reload|restart}" 86 | exit 2 87 | 88 | esac 89 | -------------------------------------------------------------------------------- /conf/logrotate.d/ctrl: -------------------------------------------------------------------------------- 1 | /var/log/ctrl/*.log { 2 | rotate 7 3 | daily 4 | compress 5 | missingok 6 | delaycompress 7 | copytruncate 8 | } 9 | -------------------------------------------------------------------------------- /conf/nginx/localhost-ssl.conf: -------------------------------------------------------------------------------- 1 | upstream ctrl { 2 | server 127.0.0.1:9691; 3 | } 4 | 5 | #upstream ctrl-ssl { 6 | # server 127.0.0.1:9791; 7 | #} 8 | 9 | server { 10 | 11 | listen 443 ssl http2; 12 | 13 | ssl_certificate /etc/ssl/example.com/fullchain.pem; 14 | ssl_trusted_certificate /etc/ssl/example.com/chain.pem; 15 | ssl_certificate_key /etc/ssl/example.com/privkey.pem; 16 | 17 | ssl_session_timeout 60m; 18 | ssl_session_cache shared:SSL:64m; 19 | 20 | ssl_protocols TLSv1.1 TLSv1.2 TLSv1.3; 21 | 22 | server_name example.com; 23 | 24 | resolver 127.0.0.1 8.8.8.8 8.8.4.4; 25 | 26 | error_log /var/log/nginx/example.com.error.ctrl.log; 27 | 28 | proxy_connect_timeout 5; 29 | proxy_send_timeout 900; 30 | proxy_read_timeout 900; 31 | lingering_time 30; 32 | lingering_timeout 5; 33 | client_header_timeout 5; 34 | client_body_timeout 900; 35 | send_timeout 900; 36 | keepalive_timeout 900; 37 | reset_timedout_connection on; 38 | 39 | # add_header Access-Control-Allow-Origin '*' always; 40 | add_header Access-Control-Allow-Methods 'GET, OPTIONS, POST' always; 41 | 42 | rewrite ^/(.*)/$ /$1 permanent; 43 | 44 | if ($request_method = 'OPTIONS') { 45 | return 200; 46 | } 47 | 48 | client_max_body_size 512m; 49 | 50 | location / { 51 | 52 | access_log /var/log/nginx/example.com.access.ctrl.log; 53 | 54 | proxy_set_header Host localhost; 55 | proxy_pass http://ctrl; 56 | # proxy_pass https://ctrl; 57 | proxy_set_header X-Real-IP $remote_addr; 58 | proxy_set_header X-Forwarded-Proto http; 59 | # proxy_set_header X-Forwarded-Proto https; 60 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 61 | 62 | } 63 | 64 | } -------------------------------------------------------------------------------- /conf/nginx/localhost.conf: -------------------------------------------------------------------------------- 1 | upstream ctrl { 2 | server 127.0.0.1:9691; 3 | } 4 | 5 | server { 6 | 7 | listen 80; 8 | 9 | server_name _; 10 | 11 | error_log /var/log/nginx/localhost.error.ctrl.log; 12 | 13 | proxy_connect_timeout 5; 14 | proxy_send_timeout 900; 15 | proxy_read_timeout 900; 16 | lingering_time 30; 17 | lingering_timeout 5; 18 | client_header_timeout 5; 19 | client_body_timeout 900; 20 | send_timeout 900; 21 | keepalive_timeout 900; 22 | reset_timedout_connection on; 23 | 24 | # add_header Access-Control-Allow-Origin '*' always; 25 | add_header Access-Control-Allow-Methods 'GET, OPTIONS, POST' always; 26 | 27 | rewrite ^/(.*)/$ /$1 permanent; 28 | 29 | if ($request_method = 'OPTIONS') { 30 | return 200; 31 | } 32 | 33 | client_max_body_size 512m; 34 | 35 | location / { 36 | 37 | access_log /var/log/nginx/localhost.access.ctrl.log; 38 | 39 | proxy_set_header Host localhost; 40 | proxy_pass http://ctrl; 41 | proxy_set_header X-Real-IP $remote_addr; 42 | proxy_set_header X-Forwarded-Proto http; 43 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 44 | 45 | } 46 | 47 | } -------------------------------------------------------------------------------- /conf/systemd/ctrl.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=cTRL Service 3 | 4 | [Service] 5 | Type=simple 6 | User=ctrl 7 | Group=ctrl 8 | PIDFile=/run/ctrl/ctrl.pid 9 | RuntimeDirectory=ctrl 10 | TimeoutStopSec=86400 11 | 12 | ExecStart=/usr/sbin/ctrl 13 | ExecStop=/bin/kill -s TERM $MAINPID 14 | 15 | Restart=always 16 | RestartSec=60s 17 | 18 | StandardOutput=syslog 19 | StandardError=syslog 20 | 21 | LimitNOFILE=131072 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | Alias=ctrl.service 26 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "bytes" 29 | "github.com/xujiajun/nutsdb" 30 | "io/ioutil" 31 | ) 32 | 33 | // DB Helpers 34 | 35 | // NDBInsert : NutsDB insert key function 36 | func NDBInsert(db *nutsdb.DB, bucket string, key []byte, value []byte, ttl uint32) error { 37 | 38 | nerr := db.Update(func(tx *nutsdb.Tx) error { 39 | 40 | err := tx.Put(bucket, key, value, ttl) 41 | if err != nil { 42 | return err 43 | } 44 | 45 | return nil 46 | 47 | }) 48 | 49 | if nerr != nil { 50 | return nerr 51 | } 52 | 53 | return nil 54 | 55 | } 56 | 57 | // NDBDelete : NutsDB delete key function 58 | func NDBDelete(db *nutsdb.DB, bucket string, key []byte) error { 59 | 60 | nerr := db.Update(func(tx *nutsdb.Tx) error { 61 | 62 | err := tx.Delete(bucket, key) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | return nil 68 | 69 | }) 70 | 71 | if nerr != nil { 72 | return nerr 73 | } 74 | 75 | return nil 76 | 77 | } 78 | 79 | // NDBMerge : NutsDB merge compaction function 80 | func NDBMerge(db *nutsdb.DB, dir string) error { 81 | 82 | var err error 83 | 84 | err = db.Merge() 85 | if err != nil { 86 | return err 87 | } 88 | 89 | segs, err := ioutil.ReadDir(dir) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | for _, seg := range segs { 95 | 96 | sn := seg.Name() 97 | ss := seg.Size() 98 | 99 | fn := dir + "/" + sn 100 | 101 | emptyBuffer := make([]byte, ss) 102 | 103 | segmentBuffer, err := ioutil.ReadFile(fn) 104 | if err != nil { 105 | return err 106 | } 107 | 108 | if segmentBuffer != nil { 109 | 110 | if bytes.Equal(emptyBuffer, segmentBuffer) { 111 | err = RemoveSegment(fn) 112 | if err != nil { 113 | return err 114 | } 115 | } 116 | 117 | } 118 | 119 | } 120 | 121 | return nil 122 | 123 | } 124 | -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "os" 29 | "path/filepath" 30 | ) 31 | 32 | // File Helpers 33 | 34 | // FileExists : check existence of requested file 35 | func FileExists(filename string) bool { 36 | 37 | if fi, err := os.Stat(filename); err == nil { 38 | 39 | if fi.Mode().IsRegular() { 40 | return true 41 | } 42 | 43 | } 44 | 45 | return false 46 | 47 | } 48 | 49 | // FileOrLinkExists : check existence of requested file or symlink 50 | func FileOrLinkExists(filename string) bool { 51 | 52 | if fi, err := os.Stat(filename); err == nil { 53 | 54 | if fi.Mode().IsRegular() { 55 | return true 56 | } 57 | 58 | } 59 | 60 | if _, err := filepath.EvalSymlinks(filename); err == nil { 61 | return true 62 | } 63 | 64 | return false 65 | 66 | } 67 | 68 | // DirExists : check existence of requested directory 69 | func DirExists(filename string) bool { 70 | 71 | if fi, err := os.Stat(filename); err == nil { 72 | 73 | if fi.Mode().IsDir() { 74 | return true 75 | } 76 | 77 | } 78 | 79 | return false 80 | 81 | } 82 | 83 | // RemoveSegment : remove NutsDB segment file if empty 84 | func RemoveSegment(filename string) error { 85 | 86 | err := os.Remove(filename) 87 | if err != nil { 88 | return err 89 | } 90 | 91 | return nil 92 | 93 | } 94 | -------------------------------------------------------------------------------- /get.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "bytes" 29 | "crypto/sha512" 30 | "encoding/gob" 31 | "errors" 32 | "fmt" 33 | "github.com/eltaline/mmutex" 34 | "github.com/kataras/iris/v12" 35 | "github.com/xujiajun/nutsdb" 36 | "net" 37 | "strings" 38 | "sync" 39 | ) 40 | 41 | // Get 42 | 43 | // CtrlShow : Show task command or commands method 44 | func CtrlShow(cldb *nutsdb.DB, wg *sync.WaitGroup) iris.Handler { 45 | return func(ctx iris.Context) { 46 | defer wg.Done() 47 | 48 | var err error 49 | 50 | // Wait Group 51 | 52 | wg.Add(1) 53 | 54 | // Loggers 55 | 56 | getLogger, getlogfile := GetLogger() 57 | defer getlogfile.Close() 58 | 59 | // Shutdown 60 | 61 | if shutdown { 62 | ctx.StatusCode(iris.StatusInternalServerError) 63 | return 64 | } 65 | 66 | // Variables 67 | 68 | errempty := errors.New("bucket is empty") 69 | errscans := errors.New("prefix scans not found") 70 | errsearchscans := errors.New("prefix and search scans not found") 71 | 72 | bprefix := []byte("t:") 73 | 74 | var ftsk []FullTask 75 | var f FullTask 76 | var resp []GetTask 77 | var p GetTask 78 | 79 | // IP Client 80 | 81 | ip := ctx.RemoteAddr() 82 | cip := net.ParseIP(ip) 83 | ush := ctx.GetHeader("Auth") 84 | vhost := strings.Split(ctx.Host(), ":")[0] 85 | 86 | key := ctx.URLParam("key") 87 | ttype := ctx.URLParam("type") 88 | queue := ctx.URLParam("queue") 89 | 90 | params := ctx.URLParams() 91 | 92 | badhost := true 93 | baduser := true 94 | badip := true 95 | 96 | user := "" 97 | pass := "" 98 | phsh := "" 99 | 100 | rvbucket := "" 101 | wvbucket := "" 102 | fvbucket := "" 103 | 104 | if ush != "" { 105 | 106 | mchpair := rgxpair.MatchString(ush) 107 | 108 | if !mchpair { 109 | 110 | ctx.StatusCode(iris.StatusBadRequest) 111 | 112 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | Bad authorization format", vhost, ip) 113 | 114 | if debugmode { 115 | 116 | _, err = ctx.Writef("[ERRO] Bad authorization format | Virtual Host [%s]\n", vhost) 117 | if err != nil { 118 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 119 | } 120 | 121 | } 122 | 123 | return 124 | 125 | } 126 | 127 | sha_512 := sha512.New() 128 | 129 | user = strings.Split(ush, ":")[0] 130 | pass = strings.Split(ush, ":")[1] 131 | 132 | sha_512.Write([]byte(pass)) 133 | phsh = fmt.Sprintf("%x", sha_512.Sum(nil)) 134 | 135 | } 136 | 137 | for _, Server := range config.Server { 138 | 139 | if vhost == Server.HOST { 140 | 141 | badhost = false 142 | 143 | if user != "" && pass != "" { 144 | 145 | for _, Vhost := range ussallow { 146 | 147 | if vhost == Vhost.Vhost { 148 | 149 | for _, PAIR := range Vhost.PAIR { 150 | 151 | if user == PAIR.User && phsh == PAIR.Hash { 152 | baduser = false 153 | break 154 | } 155 | } 156 | 157 | break 158 | 159 | } 160 | 161 | } 162 | 163 | } 164 | 165 | if baduser { 166 | 167 | for _, Vhost := range ipsallow { 168 | 169 | if vhost == Vhost.Vhost { 170 | 171 | for _, CIDR := range Vhost.CIDR { 172 | _, ipnet, _ := net.ParseCIDR(CIDR.Addr) 173 | if ipnet.Contains(cip) { 174 | badip = false 175 | break 176 | } 177 | } 178 | 179 | break 180 | 181 | } 182 | 183 | } 184 | 185 | } 186 | 187 | rvbucket = "recv" + "_" + vhost + ":" 188 | wvbucket = "work" + "_" + vhost + ":" 189 | fvbucket = "comp" + "_" + vhost + ":" 190 | 191 | break 192 | 193 | } 194 | 195 | } 196 | 197 | if badhost { 198 | 199 | ctx.StatusCode(iris.StatusMisdirectedRequest) 200 | 201 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 421 | Not found configured virtual host", vhost, ip) 202 | 203 | if debugmode { 204 | 205 | _, err = ctx.Writef("[ERRO] Not found configured virtual host | Virtual Host [%s]\n", vhost) 206 | if err != nil { 207 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 208 | } 209 | 210 | } 211 | 212 | return 213 | 214 | } 215 | 216 | if baduser && badip { 217 | 218 | ctx.StatusCode(iris.StatusForbidden) 219 | 220 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 403 | Forbidden", vhost, ip) 221 | 222 | if debugmode { 223 | 224 | _, err = ctx.Writef("[ERRO] Not found allowed user or not found allowed ip | Virtual Host [%s]\n", vhost) 225 | if err != nil { 226 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 227 | } 228 | 229 | } 230 | 231 | return 232 | 233 | } 234 | 235 | if len(params) == 0 { 236 | 237 | ctx.StatusCode(iris.StatusForbidden) 238 | 239 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 403 | The query arguments is not given during GET request", vhost, ip) 240 | 241 | if debugmode { 242 | 243 | _, err = ctx.WriteString("[ERRO] The query arguments is not given during GET request\n") 244 | if err != nil { 245 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 246 | } 247 | 248 | } 249 | 250 | return 251 | 252 | } 253 | 254 | if queue != "" { 255 | 256 | cerr := cldb.View(func(tx *nutsdb.Tx) error { 257 | 258 | var err error 259 | 260 | var tasks nutsdb.Entries 261 | 262 | var rgxkey string 263 | 264 | switch { 265 | 266 | case key != "" && ttype != "" && queue == "received": 267 | 268 | rgxkey = "(.+" + ttype + ":" + key + ")$" 269 | 270 | tasks, _, err = tx.PrefixSearchScan(rvbucket, bprefix, rgxkey, -1, -1) 271 | if tasks == nil { 272 | return nil 273 | } 274 | 275 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 276 | return nil 277 | } 278 | 279 | if err != nil { 280 | return err 281 | } 282 | 283 | case key != "" && ttype != "" && queue == "working": 284 | 285 | rgxkey = "(.+" + ttype + ":" + key + ")$" 286 | 287 | tasks, _, err = tx.PrefixSearchScan(wvbucket, bprefix, rgxkey, -1, -1) 288 | if tasks == nil { 289 | return nil 290 | } 291 | 292 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 293 | return nil 294 | } 295 | 296 | if err != nil { 297 | return err 298 | } 299 | 300 | case key != "" && ttype != "" && queue == "completed": 301 | 302 | rgxkey = "(.+" + ttype + ":" + key + ")$" 303 | 304 | tasks, _, err = tx.PrefixSearchScan(fvbucket, bprefix, rgxkey, -1, -1) 305 | if tasks == nil { 306 | return nil 307 | } 308 | 309 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 310 | return nil 311 | } 312 | 313 | if err != nil { 314 | return err 315 | } 316 | 317 | case key != "" && queue == "received": 318 | 319 | rgxkey = "(.+" + ":" + key + ")$" 320 | 321 | tasks, _, err = tx.PrefixSearchScan(rvbucket, bprefix, rgxkey, -1, -1) 322 | if tasks == nil { 323 | return nil 324 | } 325 | 326 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 327 | return nil 328 | } 329 | 330 | if err != nil { 331 | return err 332 | } 333 | 334 | case key != "" && queue == "working": 335 | 336 | rgxkey = "(.+" + ":" + key + ")$" 337 | 338 | tasks, _, err = tx.PrefixSearchScan(wvbucket, bprefix, rgxkey, -1, -1) 339 | 340 | if tasks == nil { 341 | return nil 342 | } 343 | 344 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 345 | return nil 346 | } 347 | 348 | if err != nil { 349 | return err 350 | } 351 | 352 | case key != "" && queue == "completed": 353 | 354 | rgxkey = "(.+" + ":" + key + ")$" 355 | 356 | tasks, _, err = tx.PrefixSearchScan(fvbucket, bprefix, rgxkey, -1, -1) 357 | 358 | if tasks == nil { 359 | return nil 360 | } 361 | 362 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 363 | return nil 364 | } 365 | 366 | if err != nil { 367 | return err 368 | } 369 | 370 | case ttype != "" && queue == "received": 371 | 372 | rgxkey = "(.+" + ":" + ttype + ":" + ".+)" 373 | 374 | tasks, _, err = tx.PrefixSearchScan(rvbucket, bprefix, rgxkey, -1, -1) 375 | if tasks == nil { 376 | return nil 377 | } 378 | 379 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 380 | return nil 381 | } 382 | 383 | if err != nil { 384 | return err 385 | } 386 | 387 | case ttype != "" && queue == "working": 388 | 389 | rgxkey = "(.+" + ":" + ttype + ":" + ".+)" 390 | 391 | tasks, _, err = tx.PrefixSearchScan(wvbucket, bprefix, rgxkey, -1, -1) 392 | 393 | if tasks == nil { 394 | return nil 395 | } 396 | 397 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 398 | return nil 399 | } 400 | 401 | if err != nil { 402 | return err 403 | } 404 | 405 | case ttype != "" && queue == "completed": 406 | 407 | rgxkey = "(.+" + ":" + ttype + ":" + ".+)" 408 | 409 | tasks, _, err = tx.PrefixSearchScan(fvbucket, bprefix, rgxkey, -1, -1) 410 | 411 | if tasks == nil { 412 | return nil 413 | } 414 | 415 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 416 | return nil 417 | } 418 | 419 | if err != nil { 420 | return err 421 | } 422 | 423 | case queue == "received": 424 | tasks, err = tx.GetAll(rvbucket) 425 | 426 | if tasks == nil { 427 | return nil 428 | } 429 | 430 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 431 | return nil 432 | } 433 | 434 | if err != nil { 435 | return err 436 | } 437 | 438 | case queue == "working": 439 | tasks, err = tx.GetAll(wvbucket) 440 | 441 | if tasks == nil { 442 | return nil 443 | } 444 | 445 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 446 | return nil 447 | } 448 | 449 | if err != nil { 450 | return err 451 | } 452 | 453 | case queue == "completed": 454 | tasks, err = tx.GetAll(fvbucket) 455 | 456 | if tasks == nil { 457 | return nil 458 | } 459 | 460 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 461 | return nil 462 | } 463 | 464 | if err != nil { 465 | return err 466 | } 467 | 468 | default: 469 | 470 | return nil 471 | 472 | } 473 | 474 | for _, rtask := range tasks { 475 | 476 | var rt RawTask 477 | 478 | rtdec := gob.NewDecoder(bytes.NewReader(rtask.Value)) 479 | err := rtdec.Decode(&rt) 480 | if err != nil { 481 | 482 | ctx.StatusCode(iris.StatusInternalServerError) 483 | 484 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Gob decode from db error | %v", vhost, ip, err) 485 | 486 | if debugmode { 487 | 488 | _, err = ctx.WriteString("[ERRO] Gob decode from db error\n") 489 | if err != nil { 490 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 491 | } 492 | 493 | } 494 | 495 | } 496 | 497 | f.Key = rtask.Key 498 | f.Time = rt.Time 499 | f.Type = rt.Type 500 | f.Path = rt.Path 501 | f.Lock = rt.Lock 502 | f.Command = rt.Command 503 | f.Threads = rt.Threads 504 | f.Timeout = rt.Timeout 505 | f.Ttltime = rt.Ttltime 506 | f.Interval = rt.Interval 507 | f.Repeaterr = rt.Repeaterr 508 | f.Repeatcnt = rt.Repeatcnt 509 | f.Interr = rt.Interr 510 | f.Intcnt = rt.Intcnt 511 | f.Lookout = rt.Lookout 512 | f.Replace = rt.Replace 513 | f.Stdcode = rt.Stdcode 514 | f.Stdout = rt.Stdout 515 | f.Errcode = rt.Errcode 516 | f.Stderr = rt.Stderr 517 | f.Runtime = rt.Runtime 518 | 519 | ftsk = append(ftsk, f) 520 | 521 | } 522 | 523 | return nil 524 | 525 | }) 526 | 527 | if len(ftsk) == 0 { 528 | 529 | _, err = ctx.WriteString("[]") 530 | if err != nil { 531 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 532 | } 533 | 534 | return 535 | 536 | } 537 | 538 | if cerr != nil { 539 | 540 | ctx.StatusCode(iris.StatusInternalServerError) 541 | 542 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Get task from db error | Key [%s] | %v", vhost, ip, key, err) 543 | 544 | if debugmode { 545 | 546 | _, err = ctx.WriteString("[ERRO] Get task from db error\n") 547 | if err != nil { 548 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 549 | } 550 | 551 | } 552 | 553 | return 554 | 555 | } 556 | 557 | for _, task := range ftsk { 558 | 559 | spk := strings.Split(string(task.Key), ":")[4] 560 | 561 | p.Key = spk 562 | p.Time = task.Time 563 | p.Type = task.Type 564 | p.Path = task.Path 565 | p.Lock = task.Lock 566 | p.Command = task.Command 567 | p.Threads = task.Threads 568 | p.Timeout = task.Timeout 569 | p.Ttltime = task.Ttltime 570 | p.Interval = task.Interval 571 | p.Repeaterr = task.Repeaterr 572 | p.Repeatcnt = task.Repeatcnt 573 | p.Interr = task.Interr 574 | p.Intcnt = task.Intcnt 575 | p.Lookout = task.Lookout 576 | p.Replace = task.Replace 577 | p.Stdcode = task.Stdcode 578 | p.Stdout = task.Stdout 579 | p.Errcode = task.Errcode 580 | p.Stderr = task.Stderr 581 | p.Runtime = task.Runtime 582 | 583 | resp = append(resp, p) 584 | 585 | } 586 | 587 | jkeys, _ := JSONMarshal(resp, true) 588 | allkeys := string(jkeys) 589 | rbytes := []byte(allkeys) 590 | hsize := fmt.Sprintf("%d", len(rbytes)) 591 | 592 | ctx.Header("Content-Type", "application/json") 593 | ctx.Header("Content-Length", hsize) 594 | 595 | _, err = ctx.Write(rbytes) 596 | if err != nil { 597 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 598 | } 599 | 600 | return 601 | 602 | } 603 | 604 | ctx.StatusCode(iris.StatusBadRequest) 605 | 606 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The query is not supported during GET request", vhost, ip) 607 | 608 | if debugmode { 609 | 610 | _, err = ctx.WriteString("[ERRO] The query is not supported during GET request\n") 611 | if err != nil { 612 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 613 | } 614 | 615 | } 616 | 617 | } 618 | 619 | } 620 | 621 | // CtrlDel : Delete task command or commands method 622 | func CtrlDel(cldb *nutsdb.DB, keymutex *mmutex.Mutex, wg *sync.WaitGroup) iris.Handler { 623 | return func(ctx iris.Context) { 624 | defer wg.Done() 625 | 626 | var err error 627 | 628 | // Wait Group 629 | 630 | wg.Add(1) 631 | 632 | // Loggers 633 | 634 | getLogger, getlogfile := GetLogger() 635 | defer getlogfile.Close() 636 | 637 | // Shutdown 638 | 639 | if shutdown { 640 | ctx.StatusCode(iris.StatusInternalServerError) 641 | return 642 | } 643 | 644 | // Variables 645 | 646 | errempty := errors.New("bucket is empty") 647 | errscans := errors.New("prefix scans not found") 648 | errsearchscans := errors.New("prefix and search scans not found") 649 | 650 | bprefix := []byte("t:") 651 | 652 | var ftsk []FullTask 653 | var f FullTask 654 | var resp []DelTask 655 | var d DelTask 656 | 657 | bucket := "" 658 | 659 | // IP Client 660 | 661 | ip := ctx.RemoteAddr() 662 | cip := net.ParseIP(ip) 663 | ush := ctx.GetHeader("Auth") 664 | vhost := strings.Split(ctx.Host(), ":")[0] 665 | 666 | key := ctx.URLParam("key") 667 | ttype := ctx.URLParam("type") 668 | queue := ctx.URLParam("queue") 669 | 670 | params := ctx.URLParams() 671 | 672 | badhost := true 673 | baduser := true 674 | badip := true 675 | 676 | user := "" 677 | pass := "" 678 | phsh := "" 679 | 680 | rvbucket := "" 681 | wvbucket := "" 682 | fvbucket := "" 683 | 684 | if ush != "" { 685 | 686 | mchpair := rgxpair.MatchString(ush) 687 | 688 | if !mchpair { 689 | 690 | ctx.StatusCode(iris.StatusBadRequest) 691 | 692 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | Bad authorization format", vhost, ip) 693 | 694 | if debugmode { 695 | 696 | _, err = ctx.Writef("[ERRO] Bad authorization format | Virtual Host [%s]\n", vhost) 697 | if err != nil { 698 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 699 | } 700 | 701 | } 702 | 703 | return 704 | 705 | } 706 | 707 | sha_512 := sha512.New() 708 | 709 | user = strings.Split(ush, ":")[0] 710 | pass = strings.Split(ush, ":")[1] 711 | 712 | sha_512.Write([]byte(pass)) 713 | phsh = fmt.Sprintf("%x", sha_512.Sum(nil)) 714 | 715 | } 716 | 717 | for _, Server := range config.Server { 718 | 719 | if vhost == Server.HOST { 720 | 721 | badhost = false 722 | 723 | if user != "" && pass != "" { 724 | 725 | for _, Vhost := range ussallow { 726 | 727 | if vhost == Vhost.Vhost { 728 | 729 | for _, PAIR := range Vhost.PAIR { 730 | 731 | if user == PAIR.User && phsh == PAIR.Hash { 732 | baduser = false 733 | break 734 | } 735 | } 736 | 737 | break 738 | 739 | } 740 | 741 | } 742 | 743 | } 744 | 745 | if baduser { 746 | 747 | for _, Vhost := range ipsallow { 748 | 749 | if vhost == Vhost.Vhost { 750 | 751 | for _, CIDR := range Vhost.CIDR { 752 | _, ipnet, _ := net.ParseCIDR(CIDR.Addr) 753 | if ipnet.Contains(cip) { 754 | badip = false 755 | break 756 | } 757 | } 758 | 759 | break 760 | 761 | } 762 | 763 | } 764 | 765 | } 766 | 767 | rvbucket = "recv" + "_" + vhost + ":" 768 | wvbucket = "work" + "_" + vhost + ":" 769 | fvbucket = "comp" + "_" + vhost + ":" 770 | 771 | break 772 | 773 | } 774 | 775 | } 776 | 777 | if badhost { 778 | 779 | ctx.StatusCode(iris.StatusMisdirectedRequest) 780 | 781 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 421 | Not found configured virtual host", vhost, ip) 782 | 783 | if debugmode { 784 | 785 | _, err = ctx.Writef("[ERRO] Not found configured virtual host | Virtual Host [%s]\n", vhost) 786 | if err != nil { 787 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 788 | } 789 | 790 | } 791 | 792 | return 793 | 794 | } 795 | 796 | if baduser && badip { 797 | 798 | ctx.StatusCode(iris.StatusForbidden) 799 | 800 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 403 | Forbidden", vhost, ip) 801 | 802 | if debugmode { 803 | 804 | _, err = ctx.Writef("[ERRO] Not found allowed user or not found allowed ip | Virtual Host [%s]\n", vhost) 805 | if err != nil { 806 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 807 | } 808 | 809 | } 810 | 811 | return 812 | 813 | } 814 | 815 | if len(params) == 0 { 816 | 817 | ctx.StatusCode(iris.StatusForbidden) 818 | 819 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 403 | The query arguments is not given during GET request", vhost, ip) 820 | 821 | if debugmode { 822 | 823 | _, err = ctx.WriteString("[ERRO] The query arguments is not given during GET request\n") 824 | if err != nil { 825 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 826 | } 827 | 828 | } 829 | 830 | return 831 | 832 | } 833 | 834 | if queue != "" || (queue != "" && key != "") { 835 | 836 | cerr := cldb.View(func(tx *nutsdb.Tx) error { 837 | 838 | var err error 839 | 840 | var tasks nutsdb.Entries 841 | 842 | var rgxkey string 843 | 844 | switch { 845 | case key != "" && ttype != "" && queue == "received": 846 | 847 | bucket = rvbucket 848 | 849 | rgxkey = "(.+" + ttype + ":" + key + ")$" 850 | 851 | tasks, _, err = tx.PrefixSearchScan(rvbucket, bprefix, rgxkey, -1, -1) 852 | if tasks == nil { 853 | return nil 854 | } 855 | 856 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 857 | return nil 858 | } 859 | 860 | if err != nil { 861 | return err 862 | } 863 | 864 | case key != "" && ttype != "" && queue == "working": 865 | 866 | bucket = wvbucket 867 | 868 | rgxkey = "(.+" + ttype + ":" + key + ")$" 869 | 870 | tasks, _, err = tx.PrefixSearchScan(wvbucket, bprefix, rgxkey, -1, -1) 871 | 872 | if tasks == nil { 873 | return nil 874 | } 875 | 876 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 877 | return nil 878 | } 879 | 880 | if err != nil { 881 | return err 882 | } 883 | 884 | case key != "" && ttype != "" && queue == "completed": 885 | 886 | bucket = fvbucket 887 | 888 | rgxkey = "(.+" + ttype + ":" + key + ")$" 889 | 890 | tasks, _, err = tx.PrefixSearchScan(fvbucket, bprefix, rgxkey, -1, -1) 891 | 892 | if tasks == nil { 893 | return nil 894 | } 895 | 896 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 897 | return nil 898 | } 899 | 900 | if err != nil { 901 | return err 902 | } 903 | 904 | case key != "" && queue == "received": 905 | 906 | bucket = rvbucket 907 | 908 | rgxkey = "(.+" + ":" + key + ")$" 909 | 910 | tasks, _, err = tx.PrefixSearchScan(rvbucket, bprefix, rgxkey, -1, -1) 911 | if tasks == nil { 912 | return nil 913 | } 914 | 915 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 916 | return nil 917 | } 918 | 919 | if err != nil { 920 | return err 921 | } 922 | 923 | case key != "" && queue == "working": 924 | 925 | bucket = wvbucket 926 | 927 | rgxkey = "(.+" + ":" + key + ")$" 928 | 929 | tasks, _, err = tx.PrefixSearchScan(wvbucket, bprefix, rgxkey, -1, -1) 930 | 931 | if tasks == nil { 932 | return nil 933 | } 934 | 935 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 936 | return nil 937 | } 938 | 939 | if err != nil { 940 | return err 941 | } 942 | 943 | case key != "" && queue == "completed": 944 | 945 | bucket = fvbucket 946 | 947 | rgxkey = "(.+" + ":" + key + ")$" 948 | 949 | tasks, _, err = tx.PrefixSearchScan(fvbucket, bprefix, rgxkey, -1, -1) 950 | 951 | if tasks == nil { 952 | return nil 953 | } 954 | 955 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 956 | return nil 957 | } 958 | 959 | if err != nil { 960 | return err 961 | } 962 | 963 | case ttype != "" && queue == "received": 964 | 965 | bucket = rvbucket 966 | 967 | rgxkey = "(.+" + ":" + ttype + ":" + ".+)" 968 | 969 | tasks, _, err = tx.PrefixSearchScan(rvbucket, bprefix, rgxkey, -1, -1) 970 | if tasks == nil { 971 | return nil 972 | } 973 | 974 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 975 | return nil 976 | } 977 | 978 | if err != nil { 979 | return err 980 | } 981 | 982 | case ttype != "" && queue == "working": 983 | 984 | bucket = wvbucket 985 | 986 | rgxkey = "(.+" + ":" + ttype + ":" + ".+)" 987 | 988 | tasks, _, err = tx.PrefixSearchScan(wvbucket, bprefix, rgxkey, -1, -1) 989 | 990 | if tasks == nil { 991 | return nil 992 | } 993 | 994 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 995 | return nil 996 | } 997 | 998 | if err != nil { 999 | return err 1000 | } 1001 | 1002 | case ttype != "" && queue == "completed": 1003 | 1004 | bucket = fvbucket 1005 | 1006 | rgxkey = "(.+" + ":" + ttype + ":" + ".+)" 1007 | 1008 | tasks, _, err = tx.PrefixSearchScan(fvbucket, bprefix, rgxkey, -1, -1) 1009 | 1010 | if tasks == nil { 1011 | return nil 1012 | } 1013 | 1014 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 1015 | return nil 1016 | } 1017 | 1018 | if err != nil { 1019 | return err 1020 | } 1021 | 1022 | case queue == "received": 1023 | 1024 | bucket = rvbucket 1025 | 1026 | tasks, err = tx.GetAll(rvbucket) 1027 | 1028 | if tasks == nil { 1029 | return nil 1030 | } 1031 | 1032 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 1033 | return nil 1034 | } 1035 | 1036 | if err != nil { 1037 | return err 1038 | } 1039 | 1040 | case queue == "working": 1041 | 1042 | bucket = wvbucket 1043 | 1044 | tasks, err = tx.GetAll(wvbucket) 1045 | 1046 | if tasks == nil { 1047 | return nil 1048 | } 1049 | 1050 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 1051 | return nil 1052 | } 1053 | 1054 | if err != nil { 1055 | return err 1056 | } 1057 | 1058 | case queue == "completed": 1059 | 1060 | bucket = fvbucket 1061 | 1062 | tasks, err = tx.GetAll(fvbucket) 1063 | 1064 | if tasks == nil { 1065 | return nil 1066 | } 1067 | 1068 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 1069 | return nil 1070 | } 1071 | 1072 | if err != nil { 1073 | return err 1074 | } 1075 | 1076 | default: 1077 | 1078 | return nil 1079 | 1080 | } 1081 | 1082 | for _, rtask := range tasks { 1083 | 1084 | var rt RawTask 1085 | 1086 | rtdec := gob.NewDecoder(bytes.NewReader(rtask.Value)) 1087 | err := rtdec.Decode(&rt) 1088 | if err != nil { 1089 | 1090 | ctx.StatusCode(iris.StatusInternalServerError) 1091 | 1092 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Gob decode from db error | %v", vhost, ip, err) 1093 | 1094 | if debugmode { 1095 | 1096 | _, err = ctx.WriteString("[ERRO] Gob decode from db error\n") 1097 | if err != nil { 1098 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 1099 | } 1100 | 1101 | } 1102 | 1103 | } 1104 | 1105 | f.Key = rtask.Key 1106 | f.Time = rt.Time 1107 | f.Type = rt.Type 1108 | f.Path = rt.Path 1109 | f.Lock = rt.Lock 1110 | f.Command = rt.Command 1111 | f.Threads = rt.Threads 1112 | f.Timeout = rt.Timeout 1113 | f.Ttltime = rt.Ttltime 1114 | f.Interval = rt.Interval 1115 | f.Repeaterr = rt.Repeaterr 1116 | f.Repeatcnt = rt.Repeatcnt 1117 | f.Interr = rt.Interr 1118 | f.Intcnt = rt.Intcnt 1119 | f.Lookout = rt.Lookout 1120 | f.Replace = rt.Replace 1121 | f.Stdcode = rt.Stdcode 1122 | f.Stdout = rt.Stdout 1123 | f.Errcode = rt.Errcode 1124 | f.Stderr = rt.Stderr 1125 | f.Runtime = rt.Runtime 1126 | 1127 | ftsk = append(ftsk, f) 1128 | 1129 | } 1130 | 1131 | return nil 1132 | 1133 | }) 1134 | 1135 | if len(ftsk) == 0 { 1136 | 1137 | _, err = ctx.WriteString("[]") 1138 | if err != nil { 1139 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 1140 | } 1141 | 1142 | return 1143 | 1144 | } 1145 | 1146 | if cerr != nil { 1147 | 1148 | ctx.StatusCode(iris.StatusInternalServerError) 1149 | 1150 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Del task/tasks from db error | Key [%s] | %v", vhost, ip, key, err) 1151 | 1152 | if debugmode { 1153 | 1154 | _, err = ctx.WriteString("[ERRO] Del task/tasks from db error\n") 1155 | if err != nil { 1156 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 1157 | } 1158 | 1159 | } 1160 | 1161 | return 1162 | 1163 | } 1164 | 1165 | for _, task := range ftsk { 1166 | 1167 | delcode := 0 1168 | 1169 | delerr := "" 1170 | 1171 | spk := strings.Split(string(task.Key), ":")[4] 1172 | 1173 | err = NDBDelete(cldb, bucket, task.Key) 1174 | if err != nil { 1175 | delcode = 1 1176 | delerr = err.Error() 1177 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Delete task from db error | Key [%s] | %v", vhost, ip, key, err) 1178 | } 1179 | 1180 | if bucket == wvbucket { 1181 | 1182 | skey := string(task.Key) 1183 | 1184 | cmkey := vhost + ":" + skey 1185 | 1186 | if keymutex.IsLock(cmkey) { 1187 | keymutex.UnLock(cmkey) 1188 | } 1189 | 1190 | } 1191 | 1192 | d.Key = spk 1193 | d.Time = task.Time 1194 | d.Type = task.Type 1195 | d.Path = task.Path 1196 | d.Lock = task.Lock 1197 | d.Command = task.Command 1198 | d.Threads = task.Threads 1199 | d.Timeout = task.Timeout 1200 | d.Ttltime = task.Ttltime 1201 | d.Interval = task.Interval 1202 | d.Repeaterr = task.Repeaterr 1203 | d.Repeatcnt = task.Repeatcnt 1204 | d.Interr = task.Interr 1205 | d.Intcnt = task.Intcnt 1206 | d.Lookout = task.Lookout 1207 | d.Replace = task.Replace 1208 | d.Stdcode = task.Stdcode 1209 | d.Stdout = task.Stdout 1210 | d.Errcode = task.Errcode 1211 | d.Stderr = task.Stderr 1212 | d.Runtime = task.Runtime 1213 | d.Delcode = delcode 1214 | d.Delerr = delerr 1215 | 1216 | resp = append(resp, d) 1217 | 1218 | } 1219 | 1220 | jkeys, _ := JSONMarshal(resp, true) 1221 | allkeys := string(jkeys) 1222 | rbytes := []byte(allkeys) 1223 | hsize := fmt.Sprintf("%d", len(rbytes)) 1224 | 1225 | ctx.Header("Content-Type", "application/json") 1226 | ctx.Header("Content-Length", hsize) 1227 | 1228 | _, err = ctx.Write(rbytes) 1229 | if err != nil { 1230 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 1231 | } 1232 | 1233 | return 1234 | 1235 | } 1236 | 1237 | ctx.StatusCode(iris.StatusBadRequest) 1238 | 1239 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The query is not supported during GET request", vhost, ip) 1240 | 1241 | if debugmode { 1242 | 1243 | _, err = ctx.WriteString("[ERRO] The query is not supported during GET request\n") 1244 | if err != nil { 1245 | getLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 1246 | } 1247 | 1248 | } 1249 | 1250 | } 1251 | 1252 | } 1253 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module ctrl 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/BurntSushi/toml v1.2.0 // indirect 7 | github.com/CloudyKit/jet/v3 v3.0.1 // indirect 8 | github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 // indirect 9 | github.com/ajg/form v1.5.1 // indirect 10 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect 11 | github.com/eltaline/counter v0.0.0-20210626141722-b22056417112 12 | github.com/eltaline/gron v0.0.0-20160621042432-e78485adab46 13 | github.com/eltaline/mmutex v0.0.0-20210624131550-0cd205386e7d 14 | github.com/eltaline/toml v0.3.1 15 | github.com/gofrs/uuid v4.3.0+incompatible 16 | github.com/google/go-querystring v1.1.0 // indirect 17 | github.com/imkira/go-interpol v1.1.0 // indirect 18 | github.com/iris-contrib/jade v1.1.4 // indirect 19 | github.com/iris-contrib/schema v0.0.6 // indirect 20 | github.com/json-iterator/go v1.1.12 // indirect 21 | github.com/kataras/golog v0.1.7 22 | github.com/kataras/iris/v12 v12.1.8 23 | github.com/kataras/pio v0.0.11 // indirect 24 | github.com/kataras/sitemap v0.0.6 // indirect 25 | github.com/klauspost/compress v1.15.11 // indirect 26 | github.com/microcosm-cc/bluemonday v1.0.21 // indirect 27 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 28 | github.com/moul/http2curl v1.0.0 // indirect 29 | github.com/pieterclaerhout/go-waitgroup v1.0.7 30 | github.com/robfig/cron/v3 v3.0.1 31 | github.com/roylee0704/gron v0.0.0-20160621042432-e78485adab46 // indirect 32 | github.com/ryanuber/columnize v2.1.2+incompatible // indirect 33 | github.com/sergi/go-diff v1.2.0 // indirect 34 | github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect 35 | github.com/smartystreets/goconvey v1.6.4 // indirect 36 | github.com/valyala/fasthttp v1.30.0 // indirect 37 | github.com/xeipuuv/gojsonschema v1.2.0 // indirect 38 | github.com/xujiajun/nutsdb v0.10.0 39 | github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 // indirect 40 | github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect 41 | github.com/yudai/gojsondiff v1.0.0 // indirect 42 | github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect 43 | github.com/zhangyunhao116/skipmap v0.7.0 44 | golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b // indirect 45 | golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect 46 | golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect 47 | golang.org/x/text v0.3.7 // indirect 48 | gopkg.in/ini.v1 v1.67.0 // indirect 49 | gopkg.in/yaml.v3 v3.0.1 // indirect 50 | ) 51 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= 2 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= 5 | github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 6 | github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= 7 | github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 8 | github.com/BurntSushi/toml v1.2.0 h1:Rt8g24XnyGTyglgET/PRUNlrUeu9F5L+7FilkXfZgs0= 9 | github.com/BurntSushi/toml v1.2.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= 10 | github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4slttB4vD+b9btVEnWgL3Q00OBTzVT8B9C0c= 11 | github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= 12 | github.com/CloudyKit/jet/v3 v3.0.0 h1:1PwO5w5VCtlUUl+KTOBsTGZlhjWkcybsGaAau52tOy8= 13 | github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= 14 | github.com/CloudyKit/jet/v3 v3.0.1 h1:LUBMIJtW92Fqi+fOqXbGsT/xKiwNWjYktaNAASPE7E4= 15 | github.com/CloudyKit/jet/v3 v3.0.1/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= 16 | github.com/Joker/hpp v1.0.0 h1:65+iuJYdRXv/XyN62C1uEmmOx3432rNG/rKlX6V7Kkc= 17 | github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= 18 | github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398 h1:WDC6ySpJzbxGWFh4aMxFFC28wwGp5pEuoTtvA4q/qQ4= 19 | github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= 20 | github.com/Shopify/goreferrer v0.0.0-20210630161223-536fa16abd6f h1:XeOBnoBP7K19tMBEKeUo1NOxOO+h5FFi2HGzQvvkb44= 21 | github.com/Shopify/goreferrer v0.0.0-20210630161223-536fa16abd6f/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= 22 | github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06 h1:KkH3I3sJuOLP3TjA/dfr4NAY8bghDwnXiU7cTKxQqo0= 23 | github.com/Shopify/goreferrer v0.0.0-20220729165902-8cddb4f5de06/go.mod h1:7erjKLwalezA0k99cWs5L11HWOAPNjdUZ6RxH1BXbbM= 24 | github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= 25 | github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= 26 | github.com/andybalholm/brotli v1.0.2 h1:JKnhI/XQ75uFBTiuzXpzFrUriDPiZjlOSzh6wXogP0E= 27 | github.com/andybalholm/brotli v1.0.2/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= 28 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 29 | github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= 30 | github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= 31 | github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= 32 | github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible h1:Ppm0npCCsmuR9oQaBtRuZcmILVE74aXE+AmrJj8L2ns= 33 | github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= 34 | github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= 35 | github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= 36 | github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 37 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 38 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 39 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 40 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 41 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 42 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 43 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 44 | github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= 45 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 46 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 47 | github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385 h1:clC1lXBpe2kTj2VHdaIu9ajZQe4kcEY9j0NsnDDBZ3o= 48 | github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= 49 | github.com/eltaline/counter v0.0.0-20210626141722-b22056417112 h1:M+ahAO7D6J2MUI0WB/ngk/5rtKcGZ6hLrbi9JaR/zF0= 50 | github.com/eltaline/counter v0.0.0-20210626141722-b22056417112/go.mod h1:/ReZOEsok4att1TS04vU0dFlvaL/NMNIojdW05ja8v8= 51 | github.com/eltaline/gron v0.0.0-20160621042432-e78485adab46 h1:V2wnUaQJIuAI/rjnd8UShMKch5bolYleiInMppFxCf8= 52 | github.com/eltaline/gron v0.0.0-20160621042432-e78485adab46/go.mod h1:BZBh2a0bCl2Ds8OYwAD173GsZrr49w1YxELRN4Dw5QU= 53 | github.com/eltaline/mmutex v0.0.0-20210624131550-0cd205386e7d h1:OTukNqQO8sKPMVHOVtcuAJ5D0qudI46q9jAFl0Yw5jE= 54 | github.com/eltaline/mmutex v0.0.0-20210624131550-0cd205386e7d/go.mod h1:cS8orNzYn+vncz6iN5B94+ErBPqlrsHZ1uwCFRB7+ZQ= 55 | github.com/eltaline/toml v0.3.1 h1:ww2IODjJahBmlJ2GEHpONlZQCgxXzgwVSg/MVDhA0/A= 56 | github.com/eltaline/toml v0.3.1/go.mod h1:ykmr1lXTBF+D107RP4qh9XTXVDe0c17PMWhPbNHU5RA= 57 | github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= 58 | github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= 59 | github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= 60 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 61 | github.com/gavv/httpexpect v2.0.0+incompatible h1:1X9kcRshkSKEjNJJxX9Y9mQ5BRfbxU5kORdjhlA1yX8= 62 | github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= 63 | github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= 64 | github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= 65 | github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= 66 | github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= 67 | github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= 68 | github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= 69 | github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 70 | github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= 71 | github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 72 | github.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc= 73 | github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 74 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 75 | github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 76 | github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= 77 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 78 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 79 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 80 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 81 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 82 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 83 | github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= 84 | github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= 85 | github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= 86 | github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 87 | github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= 88 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 89 | github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= 90 | github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= 91 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 92 | github.com/iris-contrib/blackfriday v2.0.0+incompatible h1:o5sHQHHm0ToHUlAJSTjW9UWicjJSDDauOOQ2AHuIVp4= 93 | github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= 94 | github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= 95 | github.com/iris-contrib/jade v1.1.3 h1:p7J/50I0cjo0wq/VWVCDFd8taPJbuFC+bq23SniRFX0= 96 | github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= 97 | github.com/iris-contrib/jade v1.1.4 h1:WoYdfyJFfZIUgqNAeOyRfTNQZOksSlZ6+FnXR3AEpX0= 98 | github.com/iris-contrib/jade v1.1.4/go.mod h1:EDqR+ur9piDl6DUgs6qRrlfzmlx/D5UybogqrXvJTBE= 99 | github.com/iris-contrib/pongo2 v0.0.1 h1:zGP7pW51oi5eQZMIlGA3I+FHY9/HOQWDB+572yin0to= 100 | github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= 101 | github.com/iris-contrib/schema v0.0.1 h1:10g/WnoRR+U+XXHWKBHeNy/+tZmM2kcAVGLOsz+yaDA= 102 | github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= 103 | github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= 104 | github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= 105 | github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= 106 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 107 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 108 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 109 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 110 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 111 | github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= 112 | github.com/kataras/golog v0.1.7 h1:0TY5tHn5L5DlRIikepcaRR/6oInIr9AiWsxzt0vvlBE= 113 | github.com/kataras/golog v0.1.7/go.mod h1:jOSQ+C5fUqsNSwurB/oAHq1IFSb0KI3l6GMa7xB6dZA= 114 | github.com/kataras/iris/v12 v12.1.8 h1:O3gJasjm7ZxpxwTH8tApZsvf274scSGQAUpNe47c37U= 115 | github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= 116 | github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= 117 | github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= 118 | github.com/kataras/pio v0.0.10 h1:b0qtPUqOpM2O+bqa5wr2O6dN4cQNwSmFd6HQqgVae0g= 119 | github.com/kataras/pio v0.0.10/go.mod h1:gS3ui9xSD+lAUpbYnjOGiQyY7sUMJO+EHpiRzhtZ5no= 120 | github.com/kataras/pio v0.0.11 h1:kqreJ5KOEXGMwHAWHDwIl+mjfNCPhAwZPa8gK7MKlyw= 121 | github.com/kataras/pio v0.0.11/go.mod h1:38hH6SWH6m4DKSYmRhlrCJ5WItwWgCVrTNU62XZyUvI= 122 | github.com/kataras/sitemap v0.0.5 h1:4HCONX5RLgVy6G4RkYOV3vKNcma9p236LdGOipJsaFE= 123 | github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= 124 | github.com/kataras/sitemap v0.0.6 h1:w71CRMMKYMJh6LR2wTgnk5hSgjVNB9KL60n5e2KHvLY= 125 | github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4= 126 | github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= 127 | github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= 128 | github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= 129 | github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= 130 | github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 131 | github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= 132 | github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= 133 | github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= 134 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 135 | github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= 136 | github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= 137 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 138 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 139 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 140 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 141 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 142 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 143 | github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= 144 | github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= 145 | github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s= 146 | github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= 147 | github.com/microcosm-cc/bluemonday v1.0.15 h1:J4uN+qPng9rvkBZBoBb8YGR+ijuklIMpSOZZLjYpbeY= 148 | github.com/microcosm-cc/bluemonday v1.0.15/go.mod h1:ZLvAzeakRwrGnzQEvstVzVt3ZpqOF2+sdFr0Om+ce30= 149 | github.com/microcosm-cc/bluemonday v1.0.18 h1:6HcxvXDAi3ARt3slx6nTesbvorIc3QeTzBNRvWktHBo= 150 | github.com/microcosm-cc/bluemonday v1.0.18/go.mod h1:Z0r70sCuXHig8YpBzCc5eGHAap2K7e/u082ZUpDRRqM= 151 | github.com/microcosm-cc/bluemonday v1.0.20 h1:flpzsq4KU3QIYAYGV/szUat7H+GPOXR0B2JU5A1Wp8Y= 152 | github.com/microcosm-cc/bluemonday v1.0.20/go.mod h1:yfBmMi8mxvaZut3Yytv+jTXRY8mxyjJ0/kQBTElld50= 153 | github.com/microcosm-cc/bluemonday v1.0.21 h1:dNH3e4PSyE4vNX+KlRGHT5KrSvjeUkoNPwEORjffHJg= 154 | github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= 155 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 156 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 157 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= 158 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 159 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 160 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 161 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= 162 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 163 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 164 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 165 | github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs= 166 | github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= 167 | github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= 168 | github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= 169 | github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= 170 | github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= 171 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 172 | github.com/pieterclaerhout/go-waitgroup v1.0.7 h1:W9zHJzXO3SPo7Rb3E8hkk8D0gJPdP6DWENt6M08IxWE= 173 | github.com/pieterclaerhout/go-waitgroup v1.0.7/go.mod h1:TJIzZKRP4sc5AT8M0RVUwXhw/OWMxg/lW8VsQz4SRBo= 174 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 175 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 176 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 177 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 178 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 179 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 180 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 181 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 182 | github.com/roylee0704/gron v0.0.0-20160621042432-e78485adab46 h1:dp1iW1JOTY63249ZTwxzwN0EKG6EvuPdfMohKo4EomY= 183 | github.com/roylee0704/gron v0.0.0-20160621042432-e78485adab46/go.mod h1:MDhl6ujYU3dbEiGclVk5uA4pHEjTS659POKAtiAWB94= 184 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 185 | github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s= 186 | github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 187 | github.com/ryanuber/columnize v2.1.2+incompatible h1:C89EOx/XBWwIXl8wm8OPJBd7kPF25UfsK2X7Ph/zCAk= 188 | github.com/ryanuber/columnize v2.1.2+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= 189 | github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk= 190 | github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= 191 | github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= 192 | github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 193 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 194 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 195 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 196 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 197 | github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= 198 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 199 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 200 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 201 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 202 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 203 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 204 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 205 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 206 | github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 207 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 208 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 209 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 210 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 211 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 212 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 213 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 214 | github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= 215 | github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= 216 | github.com/valyala/fasthttp v1.30.0 h1:nBNzWrgZUUHohyLPU/jTvXdhrcaf2m5k3bWk+3Q049g= 217 | github.com/valyala/fasthttp v1.30.0/go.mod h1:2rsYD01CKFrjjsvFxx75KlEUNpWNBY9JWD3K/7o2Cus= 218 | github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= 219 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= 220 | github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= 221 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= 222 | github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= 223 | github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= 224 | github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= 225 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 226 | github.com/xujiajun/gorouter v1.2.0/go.mod h1:yJrIta+bTNpBM/2UT8hLOaEAFckO+m/qmR3luMIQygM= 227 | github.com/xujiajun/mmap-go v1.0.1 h1:7Se7ss1fLPPRW+ePgqGpCkfGIZzJV6JPq9Wq9iv/WHc= 228 | github.com/xujiajun/mmap-go v1.0.1/go.mod h1:CNN6Sw4SL69Sui00p0zEzcZKbt+5HtEnYUsc6BKKRMg= 229 | github.com/xujiajun/nutsdb v0.6.0 h1:voRbF4bQO6gF9xiFZ+5w/fPHgEfR7Jea1NOtU34I8yE= 230 | github.com/xujiajun/nutsdb v0.6.0/go.mod h1:Q8FXi2zeQRluPpUl/CKQ6J7u/9gcI02J6cZp3owFLyA= 231 | github.com/xujiajun/nutsdb v0.8.0 h1:BkXjifwlF3akHVoTQ8B3gdF183oq4QI0JQywazEZXkM= 232 | github.com/xujiajun/nutsdb v0.8.0/go.mod h1:KOdR/RLPELQ6611VTMsSOUBSO2s7kXPINOiA/+1dlgc= 233 | github.com/xujiajun/nutsdb v0.10.0 h1:kSxd7MyZiAVQM2I79FK74WneGI+uaHsUdak8dbjzKJc= 234 | github.com/xujiajun/nutsdb v0.10.0/go.mod h1:8ZdTTF0cEQO+wN940htfHYKswFql2iB6Osckx+GmOoU= 235 | github.com/xujiajun/utils v0.0.0-20190123093513-8bf096c4f53b h1:jKG9OiL4T4xQN3IUrhUpc1tG+HfDXppkgVcrAiiaI/0= 236 | github.com/xujiajun/utils v0.0.0-20190123093513-8bf096c4f53b/go.mod h1:AZd87GYJlUzl82Yab2kTjx1EyXSQCAfZDhpTo1SQC4k= 237 | github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 h1:w0si+uee0iAaCJO9q86T6yrhdadgcsoNuh47LrUykzg= 238 | github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235/go.mod h1:MR4+0R6A9NS5IABnIM3384FfOq8QFVnm7WDrBOhIaMU= 239 | github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= 240 | github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= 241 | github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA= 242 | github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= 243 | github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= 244 | github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= 245 | github.com/zhangyunhao116/fastrand v0.1.0 h1:NBvafu24zKd6taYBVtw9MWyxkK/HSSVnFtxnPAprasw= 246 | github.com/zhangyunhao116/fastrand v0.1.0/go.mod h1:0v5KgHho0VE6HU192HnY15de/oDS8UrbBChIFjIhBtc= 247 | github.com/zhangyunhao116/sbconv v0.2.1 h1:9Z43QFpnkYNjCrRz8UR9ShVRVQ0OG5VtLKQ3mAL5zjU= 248 | github.com/zhangyunhao116/sbconv v0.2.1/go.mod h1:pdAXGnJGNM68XNdJOfGCelkEHgrQMWSeW/2/qKjuiQQ= 249 | github.com/zhangyunhao116/skipmap v0.7.0 h1:qpm8IZYgbdTGbOmjglyQfZfCDPEJfILDEqo1uo0r++M= 250 | github.com/zhangyunhao116/skipmap v0.7.0/go.mod h1:F5TrRZ1YAVSalENORSs6OsmwdOAPHnACa78SZN+ZZwc= 251 | github.com/zhangyunhao116/wyhash v0.3.2 h1:v2VfcnTCLWv0mjFNg6t2EcZAj8Y91fG7Vb004xHauXI= 252 | github.com/zhangyunhao116/wyhash v0.3.2/go.mod h1:9okT6cr1VZK9N3Tv0I6qUYDNtQgOfcQgfMRCEXt9d8I= 253 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 254 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 255 | golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 256 | golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 257 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= 258 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= 259 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= 260 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 261 | golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29 h1:tkVvjkPTB7pnW3jnid7kNyAMPVWllTNOf/qKDze4p9o= 262 | golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 263 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM= 264 | golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 265 | golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b h1:huxqepDufQpLLIRXiVkTvnxrzJlpwmIWAObmcCcUFr0= 266 | golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= 267 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 268 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 269 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 270 | golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 271 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 272 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 273 | golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 274 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 275 | golang.org/x/net v0.0.0-20210510120150-4163338589ed h1:p9UgmWI9wKpfYmgaV/IZKGdXc5qEK45tDwwwDyjS26I= 276 | golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 277 | golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 278 | golang.org/x/net v0.0.0-20211006190231-62292e806868 h1:KlOXYy8wQWTUJYFgkUI40Lzr06ofg5IRXUK5C7qZt1k= 279 | golang.org/x/net v0.0.0-20211006190231-62292e806868/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 280 | golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 281 | golang.org/x/net v0.0.0-20220403103023-749bd193bc2b h1:vI32FkLJNAWtGD4BwkThwEy6XS7ZLLMHkSkYfF8M0W0= 282 | golang.org/x/net v0.0.0-20220403103023-749bd193bc2b/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 283 | golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= 284 | golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo= 285 | golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= 286 | golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= 287 | golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4= 288 | golang.org/x/net v0.0.0-20221004154528-8021a29435af/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= 289 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 290 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 291 | golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 292 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 293 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 294 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 295 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 296 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 297 | golang.org/x/sys v0.0.0-20210514084401-e8d321eab015 h1:hZR0X1kPW+nwyJ9xRxqZk1vx5RUObAPBdKVvXPDUH/E= 298 | golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 299 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 300 | golang.org/x/sys v0.0.0-20211006194710-c8a6f5223071 h1:PjhxBct4MZii8FFR8+oeS7QOvxKOTZXgk63EU2XpfJE= 301 | golang.org/x/sys v0.0.0-20211006194710-c8a6f5223071/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 302 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 303 | golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 304 | golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12 h1:QyVthZKMsyaQwBTJE04jdNN0Pp5Fn9Qga0mrgxyERQM= 305 | golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 306 | golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 307 | golang.org/x/sys v0.0.0-20220906165534-d0df966e6959 h1:qSa+Hg9oBe6UJXrznE+yYvW51V9UbyIj/nj/KpDigo8= 308 | golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 309 | golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 310 | golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= 311 | golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 312 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 313 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 314 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 315 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 316 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 317 | golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= 318 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 319 | golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= 320 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 321 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 322 | golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 323 | golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 324 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 325 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 326 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 327 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 328 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 329 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 330 | gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= 331 | gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 332 | gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= 333 | gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 334 | gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= 335 | gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 336 | gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= 337 | gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 338 | gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= 339 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 340 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 341 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 342 | gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM= 343 | gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 344 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 345 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 346 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 347 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 348 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 349 | -------------------------------------------------------------------------------- /hlp.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "bytes" 29 | "encoding/json" 30 | "fmt" 31 | "os" 32 | ) 33 | 34 | // Working helpers 35 | 36 | func JSONMarshal(v interface{}, safeEncoding bool) ([]byte, error) { 37 | b, err := json.Marshal(v) 38 | 39 | if safeEncoding { 40 | b = bytes.Replace(b, []byte("\\u003c"), []byte("<"), -1) 41 | b = bytes.Replace(b, []byte("\\u003e"), []byte(">"), -1) 42 | b = bytes.Replace(b, []byte("\\u0026"), []byte("&"), -1) 43 | } 44 | return b, err 45 | } 46 | 47 | // RBInt : check int32 acceptable range function and then return true or false 48 | func RBInt(i int, min int, max int) bool { 49 | 50 | switch { 51 | case i >= min && i <= max: 52 | return true 53 | default: 54 | return false 55 | } 56 | 57 | } 58 | 59 | // RBUint : check uint32 acceptable range function and then return true or false 60 | func RBUint(i uint32, min uint32, max uint32) bool { 61 | 62 | switch { 63 | case i >= min && i <= max: 64 | return true 65 | default: 66 | return false 67 | } 68 | 69 | } 70 | 71 | // Check : if received value is false, then run DoExit function 72 | func Check(bvar bool, sec string, name string, val string, perm string, ferr func(string, string, string, string)) { 73 | 74 | if !bvar { 75 | ferr(sec, name, val, perm) 76 | } 77 | 78 | } 79 | 80 | // DoExit : exit program function 81 | func DoExit(sec string, name string, val string, perm string) { 82 | fmt.Printf("Bad option value error | Section [%s] | Name [%s] | Value [%v] | Permissible Value [%s]\n", sec, name, val, perm) 83 | os.Exit(1) 84 | } 85 | -------------------------------------------------------------------------------- /images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eltaline/ctrl/7b08467ea2dab127537bbb875de8dc259fe3787d/images/logo.png -------------------------------------------------------------------------------- /images/paypal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eltaline/ctrl/7b08467ea2dab127537bbb875de8dc259fe3787d/images/paypal.png -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "fmt" 29 | "github.com/kataras/golog" 30 | "os" 31 | "path/filepath" 32 | ) 33 | 34 | // Loggers 35 | 36 | // AppLogger : logger 37 | func AppLogger() (*golog.Logger, *os.File) { 38 | 39 | appLogger := golog.New() 40 | 41 | applogfile := appLogFile() 42 | 43 | if debugmode { 44 | appLogger.SetLevel("debug") 45 | appLogger.AddOutput(applogfile) 46 | } else { 47 | appLogger.SetLevel("warn") 48 | appLogger.SetOutput(applogfile) 49 | } 50 | 51 | return appLogger, applogfile 52 | 53 | } 54 | 55 | // GetLogger : logger 56 | func GetLogger() (*golog.Logger, *os.File) { 57 | 58 | getLogger := golog.New() 59 | 60 | getlogfile := getLogFile() 61 | 62 | if debugmode { 63 | getLogger.SetLevel("debug") 64 | getLogger.AddOutput(getlogfile) 65 | } else { 66 | getLogger.SetLevel("warn") 67 | getLogger.SetOutput(getlogfile) 68 | } 69 | 70 | return getLogger, getlogfile 71 | 72 | } 73 | 74 | // PostLogger : logger 75 | func PostLogger() (*golog.Logger, *os.File) { 76 | 77 | postLogger := golog.New() 78 | 79 | postlogfile := postLogFile() 80 | 81 | if debugmode { 82 | postLogger.SetLevel("debug") 83 | postLogger.AddOutput(postlogfile) 84 | } else { 85 | postLogger.SetLevel("warn") 86 | postLogger.SetOutput(postlogfile) 87 | } 88 | 89 | return postLogger, postlogfile 90 | 91 | } 92 | 93 | // Log Paths 94 | 95 | func todayAppFilename() string { 96 | logfile := filepath.Clean(logdir + "/app.log") 97 | return logfile 98 | } 99 | 100 | func todayGetFilename() string { 101 | logfile := filepath.Clean(logdir + "/get.log") 102 | return logfile 103 | } 104 | 105 | func todayPostFilename() string { 106 | logfile := filepath.Clean(logdir + "/post.log") 107 | return logfile 108 | } 109 | 110 | // Log Files 111 | 112 | func appLogFile() *os.File { 113 | 114 | filename := todayAppFilename() 115 | applogfile, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, logmode) 116 | if err != nil { 117 | fmt.Printf("Can`t open/create 'app' log file error | File [%s] | %v", filename, err) 118 | os.Exit(1) 119 | } 120 | 121 | err = os.Chmod(filename, logmode) 122 | if err != nil { 123 | fmt.Printf("Can`t chmod log file error | File [%s] | %v", filename, err) 124 | os.Exit(1) 125 | } 126 | 127 | return applogfile 128 | } 129 | 130 | func getLogFile() *os.File { 131 | filename := todayGetFilename() 132 | getlogfile, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, logmode) 133 | if err != nil { 134 | fmt.Printf("Can`t open/create 'get' log file error | File [%s] | %v", filename, err) 135 | os.Exit(1) 136 | } 137 | 138 | err = os.Chmod(filename, logmode) 139 | if err != nil { 140 | fmt.Printf("Can`t chmod log file error | File [%s] | %v", filename, err) 141 | os.Exit(1) 142 | } 143 | 144 | return getlogfile 145 | } 146 | 147 | func postLogFile() *os.File { 148 | filename := todayPostFilename() 149 | postlogfile, err := os.OpenFile(filename, os.O_CREATE|os.O_WRONLY|os.O_APPEND, logmode) 150 | if err != nil { 151 | fmt.Printf("Can`t open/create 'post' log file error | File [%s] | %v", filename, err) 152 | os.Exit(1) 153 | } 154 | 155 | err = os.Chmod(filename, logmode) 156 | if err != nil { 157 | fmt.Printf("Can`t chmod log file error | File [%s] | %v", filename, err) 158 | os.Exit(1) 159 | } 160 | 161 | return postlogfile 162 | } 163 | -------------------------------------------------------------------------------- /post.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "bytes" 29 | "crypto/sha512" 30 | "encoding/gob" 31 | "errors" 32 | "fmt" 33 | "github.com/eltaline/counter" 34 | "github.com/eltaline/mmutex" 35 | "github.com/gofrs/uuid" 36 | "github.com/kataras/iris/v12" 37 | "github.com/pieterclaerhout/go-waitgroup" 38 | "github.com/xujiajun/nutsdb" 39 | "net" 40 | "os/exec" 41 | "strconv" 42 | "strings" 43 | "sync" 44 | "time" 45 | ) 46 | 47 | // Post 48 | 49 | // CtrlRun : Start realtime task command or commands method 50 | func CtrlRun(clsmutex *mmutex.Mutex, wg *sync.WaitGroup) iris.Handler { 51 | return func(ctx iris.Context) { 52 | defer wg.Done() 53 | 54 | var err error 55 | 56 | // Wait Group 57 | 58 | wg.Add(1) 59 | 60 | // OnClose Connection 61 | 62 | ikill := false 63 | 64 | kill := make(chan bool) 65 | ucl := uuid.Must(uuid.NewV4()) 66 | rtm := time.Now().UnixNano() 67 | 68 | cls := fmt.Sprintf("%d", rtm) + ":" + fmt.Sprintf("%x", ucl) 69 | 70 | ctx.OnClose(func() { 71 | 72 | ikill = true 73 | 74 | if !clsmutex.IsLock(cls) { 75 | kill <- true 76 | close(kill) 77 | return 78 | } 79 | 80 | clsmutex.UnLock(cls) 81 | 82 | }) 83 | 84 | // Loggers 85 | 86 | postLogger, postlogfile := PostLogger() 87 | defer postlogfile.Close() 88 | 89 | // Shutdown 90 | 91 | if shutdown { 92 | ctx.StatusCode(iris.StatusInternalServerError) 93 | clsmutex.TryLock(cls) 94 | return 95 | } 96 | 97 | // Variables 98 | 99 | rsmx := &sync.Mutex{} 100 | 101 | var body []PostTask 102 | var resp []GetTask 103 | 104 | // IP Client 105 | 106 | ip := ctx.RemoteAddr() 107 | cip := net.ParseIP(ip) 108 | ush := ctx.GetHeader("Auth") 109 | vhost := strings.Split(ctx.Host(), ":")[0] 110 | 111 | params := ctx.URLParams() 112 | length := ctx.GetHeader("Content-Length") 113 | 114 | badhost := true 115 | baduser := true 116 | badip := true 117 | 118 | user := "" 119 | pass := "" 120 | phsh := "" 121 | 122 | shell := "/bin/bash" 123 | 124 | rthreads := 0 125 | 126 | vtimeout := uint32(28800) 127 | 128 | if ush != "" { 129 | 130 | mchpair := rgxpair.MatchString(ush) 131 | 132 | if !mchpair { 133 | 134 | ctx.StatusCode(iris.StatusBadRequest) 135 | 136 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | Bad authorization format", vhost, ip) 137 | 138 | if debugmode { 139 | 140 | _, err = ctx.Writef("[ERRO] Bad authorization format | Virtual Host [%s]\n", vhost) 141 | if err != nil { 142 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 143 | } 144 | 145 | } 146 | 147 | clsmutex.TryLock(cls) 148 | return 149 | 150 | } 151 | 152 | sha_512 := sha512.New() 153 | 154 | user = strings.Split(ush, ":")[0] 155 | pass = strings.Split(ush, ":")[1] 156 | 157 | sha_512.Write([]byte(pass)) 158 | phsh = fmt.Sprintf("%x", sha_512.Sum(nil)) 159 | 160 | } 161 | 162 | for _, Server := range config.Server { 163 | 164 | if vhost == Server.HOST { 165 | 166 | badhost = false 167 | 168 | if user != "" && pass != "" { 169 | 170 | for _, Vhost := range ussallow { 171 | 172 | if vhost == Vhost.Vhost { 173 | 174 | for _, PAIR := range Vhost.PAIR { 175 | 176 | if user == PAIR.User && phsh == PAIR.Hash { 177 | baduser = false 178 | break 179 | } 180 | } 181 | 182 | break 183 | 184 | } 185 | 186 | } 187 | 188 | } 189 | 190 | if baduser { 191 | 192 | for _, Vhost := range ipsallow { 193 | 194 | if vhost == Vhost.Vhost { 195 | 196 | for _, CIDR := range Vhost.CIDR { 197 | _, ipnet, _ := net.ParseCIDR(CIDR.Addr) 198 | if ipnet.Contains(cip) { 199 | badip = false 200 | break 201 | } 202 | } 203 | 204 | break 205 | 206 | } 207 | 208 | } 209 | 210 | } 211 | 212 | shell = Server.SHELL 213 | rthreads = int(Server.RTHREADS) 214 | vtimeout = Server.VTIMEOUT 215 | 216 | break 217 | 218 | } 219 | 220 | } 221 | 222 | if badhost { 223 | 224 | ctx.StatusCode(iris.StatusMisdirectedRequest) 225 | 226 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 421 | Not found configured virtual host", vhost, ip) 227 | 228 | if debugmode { 229 | 230 | _, err = ctx.Writef("[ERRO] Not found configured virtual host | Virtual Host [%s]\n", vhost) 231 | if err != nil { 232 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 233 | } 234 | 235 | } 236 | 237 | clsmutex.TryLock(cls) 238 | return 239 | 240 | } 241 | 242 | if baduser && badip { 243 | 244 | ctx.StatusCode(iris.StatusForbidden) 245 | 246 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 403 | Forbidden", vhost, ip) 247 | 248 | if debugmode { 249 | 250 | _, err = ctx.Writef("[ERRO] Not found allowed user or not found allowed ip | Virtual Host [%s]\n", vhost) 251 | if err != nil { 252 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 253 | } 254 | 255 | } 256 | 257 | clsmutex.TryLock(cls) 258 | return 259 | 260 | } 261 | 262 | if len(params) != 0 { 263 | 264 | ctx.StatusCode(iris.StatusForbidden) 265 | 266 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 403 | The query arguments is not allowed during POST request", vhost, ip) 267 | 268 | if debugmode { 269 | 270 | _, err = ctx.WriteString("[ERRO] The query arguments is not allowed during POST request\n") 271 | if err != nil { 272 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 273 | } 274 | 275 | } 276 | 277 | clsmutex.TryLock(cls) 278 | return 279 | 280 | } 281 | 282 | clength, err := strconv.ParseInt(length, 10, 64) 283 | if err != nil { 284 | 285 | ctx.StatusCode(iris.StatusBadRequest) 286 | 287 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | Content length error during POST request | Content-Length [%s] | %v", vhost, ip, length, err) 288 | 289 | if debugmode { 290 | 291 | _, err = ctx.WriteString("[ERRO] Content length error during POST request\n") 292 | if err != nil { 293 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 294 | } 295 | 296 | } 297 | 298 | clsmutex.TryLock(cls) 299 | return 300 | 301 | } 302 | 303 | if clength == 0 { 304 | 305 | ctx.StatusCode(iris.StatusBadRequest) 306 | 307 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The body was empty during POST request | Content-Length [%s]", vhost, ip, length) 308 | 309 | if debugmode { 310 | 311 | _, err = ctx.WriteString("[ERRO] The body was empty during POST request\n") 312 | if err != nil { 313 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 314 | } 315 | 316 | } 317 | 318 | clsmutex.TryLock(cls) 319 | return 320 | 321 | } 322 | 323 | err = ctx.ReadJSON(&body) 324 | if err != nil { 325 | 326 | ctx.StatusCode(iris.StatusBadRequest) 327 | 328 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The body was bad during POST request | Content-Length [%s]", vhost, ip, length) 329 | 330 | if debugmode { 331 | 332 | _, err = ctx.Writef("[ERRO] The body was bad during POST request, %v\n", err) 333 | 334 | if err != nil { 335 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 336 | } 337 | 338 | } 339 | 340 | clsmutex.TryLock(cls) 341 | return 342 | 343 | } 344 | 345 | limit := 0 346 | 347 | GlbMap.RLock() 348 | cntvhs, ok := GlbMap.Glb[vhost] 349 | GlbMap.RUnlock() 350 | 351 | if !ok { 352 | 353 | GlbMap.Lock() 354 | 355 | cntvhs, ok = GlbMap.Glb[vhost] 356 | if !ok { 357 | 358 | GlbMap.Glb[vhost] = new(Cnts) 359 | cntvhs = GlbMap.Glb[vhost] 360 | cntvhs.Cnt = counter.NewInt64() 361 | 362 | } 363 | 364 | GlbMap.Unlock() 365 | 366 | } 367 | 368 | for { 369 | 370 | curthreads := cntvhs.Cnt.Get() 371 | 372 | if int(curthreads) < rthreads { 373 | limit = rthreads - int(curthreads) 374 | break 375 | } 376 | 377 | select { 378 | case <-kill: 379 | 380 | clsmutex.TryLock(cls) 381 | return 382 | 383 | default: 384 | 385 | time.Sleep(time.Duration(250) * time.Millisecond) 386 | 387 | } 388 | 389 | } 390 | 391 | if limit <= 0 { 392 | clsmutex.TryLock(cls) 393 | return 394 | } 395 | 396 | qwg := waitgroup.NewWaitGroup(limit) 397 | 398 | for _, task := range body { 399 | 400 | prefskey := task.Key 401 | preftype := task.Type 402 | prefpath := task.Path 403 | preflock := task.Lock 404 | prefcomm := task.Command 405 | preftout := task.Timeout 406 | 407 | qwait := make(chan bool) 408 | 409 | qwg.Add(func() { 410 | 411 | var p GetTask 412 | 413 | skey := prefskey 414 | ftmst := time.Now().Unix() 415 | ftype := preftype 416 | fpath := prefpath 417 | flock := preflock 418 | fcomm := prefcomm 419 | ftout := preftout 420 | 421 | cntvhs.Cnt.Incr() 422 | 423 | qwait <- true 424 | 425 | stdcode := 0 426 | errcode := 0 427 | 428 | stdout := "" 429 | stderr := "" 430 | 431 | if skey == "" || ftype == "" || fpath == "" || flock == "" || fcomm == "" { 432 | 433 | ctx.StatusCode(iris.StatusBadRequest) 434 | 435 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The body was not contains enough parameters during POST request", vhost, ip) 436 | 437 | if debugmode { 438 | 439 | _, err = ctx.WriteString("[ERRO] The body was not contains enough parameters during POST request\n") 440 | if err != nil { 441 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 442 | } 443 | 444 | } 445 | 446 | cntvhs.Cnt.Decr() 447 | clsmutex.TryLock(cls) 448 | return 449 | 450 | } 451 | 452 | kmchcln := rgxcln.MatchString(task.Key) 453 | 454 | if kmchcln { 455 | 456 | ctx.StatusCode(iris.StatusBadRequest) 457 | 458 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The key does not allow to contains colon(:) during POST request", vhost, ip) 459 | 460 | if debugmode { 461 | 462 | _, err = ctx.WriteString("[ERRO] The key does not allow to contains colon(:) during POST request\n") 463 | if err != nil { 464 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 465 | } 466 | 467 | } 468 | 469 | cntvhs.Cnt.Decr() 470 | clsmutex.TryLock(cls) 471 | return 472 | 473 | } 474 | 475 | tmchcln := rgxcln.MatchString(task.Type) 476 | 477 | if tmchcln { 478 | 479 | ctx.StatusCode(iris.StatusBadRequest) 480 | 481 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The type does not allow to contains colon(:) during POST request", vhost, ip) 482 | 483 | if debugmode { 484 | 485 | _, err = ctx.WriteString("[ERRO] The type does not allow to contains colon(:) during POST request\n") 486 | if err != nil { 487 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 488 | } 489 | 490 | } 491 | 492 | cntvhs.Cnt.Decr() 493 | clsmutex.TryLock(cls) 494 | return 495 | 496 | } 497 | 498 | if ftout == uint32(0) || ftout >= uint32(2592000) { 499 | ftout = vtimeout 500 | } 501 | 502 | var cmmout bytes.Buffer 503 | var cmmerr bytes.Buffer 504 | 505 | scm := shell + " -c " + "\"cd " + fpath + " " + "&&" + " " + fcomm + "\"" 506 | cmm := exec.Command(shell, "-c", scm) 507 | 508 | var rtime float64 509 | 510 | stime := time.Now() 511 | 512 | cmm.Stdout = &cmmout 513 | cmm.Stderr = &cmmerr 514 | 515 | cwg := waitgroup.NewWaitGroup(1) 516 | 517 | crun := make(chan bool) 518 | 519 | cwg.Add(func() { 520 | 521 | err = cmm.Start() 522 | if err != nil { 523 | 524 | if exitError, ok := err.(*exec.ExitError); ok { 525 | errcode = exitError.ExitCode() 526 | } else { 527 | errcode = 255 528 | } 529 | 530 | stderr = err.Error() 531 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | Start command error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, ip, skey, fpath, flock, scm, err) 532 | 533 | } 534 | 535 | err = cmm.Wait() 536 | if err != nil { 537 | 538 | if exitError, ok := err.(*exec.ExitError); ok { 539 | errcode = exitError.ExitCode() 540 | } else { 541 | errcode = 1 542 | } 543 | 544 | stderr = err.Error() 545 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | Execute command error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, ip, skey, fpath, flock, scm, err) 546 | 547 | } 548 | 549 | crun <- true 550 | 551 | }) 552 | 553 | cmmt := time.After(time.Duration(int(ftout)) * time.Second) 554 | 555 | Kill: 556 | 557 | for { 558 | 559 | if ikill { 560 | _ = cmm.Process.Kill() 561 | } 562 | 563 | select { 564 | 565 | case <-crun: 566 | break Kill 567 | case <-kill: 568 | _ = cmm.Process.Kill() 569 | <-crun 570 | break Kill 571 | case <-cmmt: 572 | _ = cmm.Process.Kill() 573 | <-crun 574 | errcode = 124 575 | break Kill 576 | 577 | default: 578 | 579 | time.Sleep(time.Duration(5) * time.Millisecond) 580 | 581 | } 582 | 583 | } 584 | 585 | cwg.Wait() 586 | close(crun) 587 | 588 | rtime = float64(time.Since(stime)) / float64(time.Millisecond) 589 | 590 | stdout = cmmout.String() 591 | stderr = cmmerr.String() 592 | 593 | p.Key = skey 594 | p.Time = ftmst 595 | p.Type = ftype 596 | p.Path = fpath 597 | p.Lock = flock 598 | p.Command = fcomm 599 | p.Timeout = ftout 600 | p.Stdcode = stdcode 601 | p.Stdout = stdout 602 | p.Errcode = errcode 603 | p.Stderr = stderr 604 | p.Runtime = rtime 605 | 606 | rsmx.Lock() 607 | resp = append(resp, p) 608 | rsmx.Unlock() 609 | 610 | cntvhs.Cnt.Decr() 611 | 612 | }) 613 | 614 | <-qwait 615 | close(qwait) 616 | 617 | } 618 | 619 | qwg.Wait() 620 | 621 | jkeys, _ := JSONMarshal(resp, true) 622 | allkeys := string(jkeys) 623 | rbytes := []byte(allkeys) 624 | hsize := fmt.Sprintf("%d", len(rbytes)) 625 | 626 | ctx.Header("Content-Type", "application/json") 627 | ctx.Header("Content-Length", hsize) 628 | 629 | _, err = ctx.Write(rbytes) 630 | if err != nil { 631 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 632 | } 633 | 634 | select { 635 | 636 | case <-kill: 637 | return 638 | default: 639 | clsmutex.TryLock(cls) 640 | 641 | } 642 | 643 | } 644 | 645 | } 646 | 647 | // CtrlTask : Start scheduled task command or commands method 648 | func CtrlTask(cldb *nutsdb.DB, wg *sync.WaitGroup) iris.Handler { 649 | return func(ctx iris.Context) { 650 | defer wg.Done() 651 | 652 | var err error 653 | 654 | // Wait Group 655 | 656 | wg.Add(1) 657 | 658 | // Loggers 659 | 660 | postLogger, postlogfile := PostLogger() 661 | defer postlogfile.Close() 662 | 663 | // Shutdown 664 | 665 | if shutdown { 666 | ctx.StatusCode(iris.StatusInternalServerError) 667 | return 668 | } 669 | 670 | // Variables 671 | 672 | errempty := errors.New("bucket is empty") 673 | errscans := errors.New("prefix scans not found") 674 | errsearchscans := errors.New("prefix and search scans not found") 675 | 676 | bprefix := []byte("t:") 677 | 678 | var body []PostTask 679 | 680 | // IP Client 681 | 682 | ip := ctx.RemoteAddr() 683 | cip := net.ParseIP(ip) 684 | ush := ctx.GetHeader("Auth") 685 | vhost := strings.Split(ctx.Host(), ":")[0] 686 | 687 | params := ctx.URLParams() 688 | length := ctx.GetHeader("Content-Length") 689 | 690 | badhost := true 691 | baduser := true 692 | badip := true 693 | 694 | user := "" 695 | pass := "" 696 | phsh := "" 697 | 698 | vthreads := uint32(1) 699 | vtimeout := uint32(28800) 700 | vttltime := uint32(86400) 701 | vinterval := uint32(0) 702 | var vrepeaterr []string 703 | vrepeatcnt := uint32(0) 704 | var vinterr []string 705 | vintcnt := uint32(0) 706 | 707 | rvbucket := "" 708 | 709 | if ush != "" { 710 | 711 | mchpair := rgxpair.MatchString(ush) 712 | 713 | if !mchpair { 714 | 715 | ctx.StatusCode(iris.StatusBadRequest) 716 | 717 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | Bad authorization format", vhost, ip) 718 | 719 | if debugmode { 720 | 721 | _, err = ctx.Writef("[ERRO] Bad authorization format | Virtual Host [%s]\n", vhost) 722 | if err != nil { 723 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 724 | } 725 | 726 | } 727 | 728 | return 729 | 730 | } 731 | 732 | sha_512 := sha512.New() 733 | 734 | user = strings.Split(ush, ":")[0] 735 | pass = strings.Split(ush, ":")[1] 736 | 737 | sha_512.Write([]byte(pass)) 738 | phsh = fmt.Sprintf("%x", sha_512.Sum(nil)) 739 | 740 | } 741 | 742 | for _, Server := range config.Server { 743 | 744 | if vhost == Server.HOST { 745 | 746 | badhost = false 747 | 748 | if user != "" && pass != "" { 749 | 750 | for _, Vhost := range ussallow { 751 | 752 | if vhost == Vhost.Vhost { 753 | 754 | for _, PAIR := range Vhost.PAIR { 755 | 756 | if user == PAIR.User && phsh == PAIR.Hash { 757 | baduser = false 758 | break 759 | } 760 | } 761 | 762 | break 763 | 764 | } 765 | 766 | } 767 | 768 | } 769 | 770 | if baduser { 771 | 772 | for _, Vhost := range ipsallow { 773 | 774 | if vhost == Vhost.Vhost { 775 | 776 | for _, CIDR := range Vhost.CIDR { 777 | _, ipnet, _ := net.ParseCIDR(CIDR.Addr) 778 | if ipnet.Contains(cip) { 779 | badip = false 780 | break 781 | } 782 | } 783 | 784 | break 785 | 786 | } 787 | 788 | } 789 | 790 | } 791 | 792 | vthreads = Server.VTHREADS 793 | vtimeout = Server.VTIMEOUT 794 | vttltime = Server.VTTLTIME 795 | vinterval = Server.VINTERVAL 796 | vrepeaterr = Server.VREPEATERR 797 | vrepeatcnt = Server.VREPEATCNT 798 | vinterr = Server.VINTERR 799 | vintcnt = Server.VINTCNT 800 | rvbucket = "recv" + "_" + vhost + ":" 801 | 802 | break 803 | 804 | } 805 | 806 | } 807 | 808 | if badhost { 809 | 810 | ctx.StatusCode(iris.StatusMisdirectedRequest) 811 | 812 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 421 | Not found configured virtual host", vhost, ip) 813 | 814 | if debugmode { 815 | 816 | _, err = ctx.Writef("[ERRO] Not found configured virtual host | Virtual Host [%s]\n", vhost) 817 | if err != nil { 818 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 819 | } 820 | 821 | } 822 | 823 | return 824 | 825 | } 826 | 827 | if baduser && badip { 828 | 829 | ctx.StatusCode(iris.StatusForbidden) 830 | 831 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 403 | Forbidden", vhost, ip) 832 | 833 | if debugmode { 834 | 835 | _, err = ctx.Writef("[ERRO] Not found allowed user or not found allowed ip | Virtual Host [%s]\n", vhost) 836 | if err != nil { 837 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 838 | } 839 | 840 | } 841 | 842 | return 843 | 844 | } 845 | 846 | if len(params) != 0 { 847 | 848 | ctx.StatusCode(iris.StatusForbidden) 849 | 850 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 403 | The query arguments is not allowed during POST request", vhost, ip) 851 | 852 | if debugmode { 853 | 854 | _, err = ctx.WriteString("[ERRO] The query arguments is not allowed during POST request\n") 855 | if err != nil { 856 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 857 | } 858 | 859 | } 860 | 861 | return 862 | 863 | } 864 | 865 | clength, err := strconv.ParseInt(length, 10, 64) 866 | if err != nil { 867 | 868 | ctx.StatusCode(iris.StatusBadRequest) 869 | 870 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | Content length error during POST request | Content-Length [%s] | %v", vhost, ip, length, err) 871 | 872 | if debugmode { 873 | 874 | _, err = ctx.WriteString("[ERRO] Content length error during POST request\n") 875 | if err != nil { 876 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 877 | } 878 | 879 | } 880 | 881 | return 882 | 883 | } 884 | 885 | if clength == 0 { 886 | 887 | ctx.StatusCode(iris.StatusBadRequest) 888 | 889 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The body was empty during POST request | Content-Length [%s]", vhost, ip, length) 890 | 891 | if debugmode { 892 | 893 | _, err = ctx.WriteString("[ERRO] The body was empty during POST request\n") 894 | if err != nil { 895 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 896 | } 897 | 898 | } 899 | 900 | return 901 | 902 | } 903 | 904 | err = ctx.ReadJSON(&body) 905 | if err != nil { 906 | 907 | ctx.StatusCode(iris.StatusBadRequest) 908 | 909 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The body was bad during POST request | Content-Length [%s]", vhost, ip, length) 910 | 911 | if debugmode { 912 | 913 | _, err = ctx.Writef("[ERRO] The body was bad during POST request, %v\n", err) 914 | 915 | if err != nil { 916 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 917 | } 918 | 919 | } 920 | 921 | return 922 | 923 | } 924 | 925 | for _, task := range body { 926 | 927 | var ftsk []FullTask 928 | var f FullTask 929 | 930 | stdcode := 0 931 | errcode := 0 932 | 933 | stdout := "" 934 | stderr := "" 935 | 936 | if task.Key == "" || task.Type == "" || task.Path == "" || task.Lock == "" || task.Command == "" { 937 | 938 | ctx.StatusCode(iris.StatusBadRequest) 939 | 940 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The body was not contains enough parameters during POST request", vhost, ip) 941 | 942 | if debugmode { 943 | 944 | _, err = ctx.WriteString("[ERRO] The body was not contains enough parameters during POST request\n") 945 | if err != nil { 946 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 947 | } 948 | 949 | } 950 | 951 | return 952 | 953 | } 954 | 955 | kmchcln := rgxcln.MatchString(task.Key) 956 | 957 | if kmchcln { 958 | 959 | ctx.StatusCode(iris.StatusBadRequest) 960 | 961 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The key does not allow to contains colon(:) during POST request", vhost, ip) 962 | 963 | if debugmode { 964 | 965 | _, err = ctx.WriteString("[ERRO] The key does not allow to contains colon(:) during POST request\n") 966 | if err != nil { 967 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 968 | } 969 | 970 | } 971 | 972 | return 973 | 974 | } 975 | 976 | tmchcln := rgxcln.MatchString(task.Type) 977 | 978 | if tmchcln { 979 | 980 | ctx.StatusCode(iris.StatusBadRequest) 981 | 982 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 400 | The type does not allow to contains colon(:) during POST request", vhost, ip) 983 | 984 | if debugmode { 985 | 986 | _, err = ctx.WriteString("[ERRO] The type does not allow to contains colon(:) during POST request\n") 987 | if err != nil { 988 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 989 | } 990 | 991 | } 992 | 993 | return 994 | 995 | } 996 | 997 | pbuffer := new(bytes.Buffer) 998 | enc := gob.NewEncoder(pbuffer) 999 | 1000 | internal := uuid.Must(uuid.NewV4()) 1001 | 1002 | ftmst := time.Now().Unix() 1003 | 1004 | bkey := []byte("t:" + fmt.Sprintf("%d", ftmst) + ":" + fmt.Sprintf("%x", internal) + ":" + task.Type + ":" + task.Key) 1005 | skey := task.Key 1006 | 1007 | ftype := task.Type 1008 | fpath := task.Path 1009 | flock := task.Lock 1010 | fcomm := task.Command 1011 | fthreads := vthreads 1012 | ftout := vtimeout 1013 | fttl := vttltime 1014 | fint := vinterval 1015 | frerr := vrepeaterr 1016 | frcnt := vrepeatcnt 1017 | fierr := vinterr 1018 | ficnt := vintcnt 1019 | 1020 | if task.Threads > uint32(0) && task.Threads <= uint32(4096) { 1021 | fthreads = task.Threads 1022 | } 1023 | 1024 | if task.Timeout > uint32(0) && task.Timeout <= uint32(2592000) { 1025 | ftout = task.Timeout 1026 | } 1027 | 1028 | if task.Ttltime > uint32(0) && task.Ttltime <= uint32(2592000) { 1029 | fttl = task.Ttltime 1030 | } 1031 | 1032 | if task.Interval > uint32(0) && task.Interval <= uint32(60) { 1033 | fint = task.Interval 1034 | } 1035 | 1036 | if len(task.Repeaterr) > 0 { 1037 | frerr = task.Repeaterr 1038 | } 1039 | 1040 | if task.Repeatcnt > uint32(0) && task.Repeatcnt <= uint32(1000) { 1041 | frcnt = task.Repeatcnt 1042 | } 1043 | 1044 | if len(task.Interr) > 0 { 1045 | fierr = task.Interr 1046 | } 1047 | 1048 | if task.Intcnt > uint32(0) && task.Intcnt <= uint32(1000) { 1049 | ficnt = task.Intcnt 1050 | } 1051 | 1052 | fsout := task.Lookout 1053 | frepl := task.Replace 1054 | 1055 | etsk := &RawTask{ 1056 | Time: ftmst, 1057 | Type: ftype, 1058 | Path: fpath, 1059 | Lock: flock, 1060 | Command: fcomm, 1061 | Threads: fthreads, 1062 | Timeout: ftout, 1063 | Ttltime: fttl, 1064 | Interval: fint, 1065 | Repeaterr: frerr, 1066 | Repeatcnt: frcnt, 1067 | Interr: fierr, 1068 | Intcnt: ficnt, 1069 | Lookout: fsout, 1070 | Replace: frepl, 1071 | Stdcode: stdcode, 1072 | Stdout: stdout, 1073 | Errcode: errcode, 1074 | Stderr: stderr, 1075 | Runtime: float64(0), 1076 | } 1077 | 1078 | err = enc.Encode(etsk) 1079 | if err != nil { 1080 | 1081 | ctx.StatusCode(iris.StatusInternalServerError) 1082 | 1083 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Gob task encode error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, ip, skey, fpath, flock, fcomm, err) 1084 | 1085 | if debugmode { 1086 | 1087 | _, err = ctx.WriteString("[ERRO] Gob task encode error\n") 1088 | if err != nil { 1089 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 1090 | } 1091 | 1092 | } 1093 | 1094 | return 1095 | 1096 | } 1097 | 1098 | if frepl { 1099 | 1100 | cerr := cldb.View(func(tx *nutsdb.Tx) error { 1101 | 1102 | var err error 1103 | 1104 | var tasks nutsdb.Entries 1105 | 1106 | rgxkey := "(.+" + ftype + ":" + skey + ")$" 1107 | 1108 | tasks, _, err = tx.PrefixSearchScan(rvbucket, bprefix, rgxkey, -1, -1) 1109 | if tasks == nil { 1110 | return nil 1111 | } 1112 | 1113 | if err != nil && (err.Error() == errempty.Error() || err.Error() == errscans.Error() || err.Error() == errsearchscans.Error()) { 1114 | return nil 1115 | } 1116 | 1117 | if err != nil { 1118 | return err 1119 | } 1120 | 1121 | for _, rtask := range tasks { 1122 | 1123 | f.Key = rtask.Key 1124 | ftsk = append(ftsk, f) 1125 | 1126 | } 1127 | 1128 | return nil 1129 | 1130 | }) 1131 | 1132 | if cerr != nil { 1133 | 1134 | ctx.StatusCode(iris.StatusInternalServerError) 1135 | 1136 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Search task/tasks in db error | Key [%s] | %v", vhost, ip, skey, err) 1137 | 1138 | if debugmode { 1139 | 1140 | _, err = ctx.WriteString("[ERRO] Search task/tasks in db error\n") 1141 | if err != nil { 1142 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 1143 | } 1144 | 1145 | } 1146 | 1147 | return 1148 | 1149 | } 1150 | 1151 | for _, task := range ftsk { 1152 | 1153 | err = NDBDelete(cldb, rvbucket, task.Key) 1154 | if err != nil { 1155 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Delete task from db error | Key [%s] | %v", vhost, ip, skey, err) 1156 | } 1157 | 1158 | } 1159 | 1160 | } 1161 | 1162 | err = NDBInsert(cldb, rvbucket, bkey, pbuffer.Bytes(), 0) 1163 | if err != nil { 1164 | 1165 | ctx.StatusCode(iris.StatusInternalServerError) 1166 | 1167 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 500 | Insert received task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, ip, skey, fpath, flock, fcomm, err) 1168 | 1169 | if debugmode { 1170 | 1171 | _, err = ctx.WriteString("[ERRO] Insert received task db error\n") 1172 | if err != nil { 1173 | postLogger.Errorf("| Virtual Host [%s] | Client IP [%s] | 499 | Can`t complete response to client | %v", vhost, ip, err) 1174 | } 1175 | 1176 | } 1177 | 1178 | return 1179 | 1180 | } 1181 | 1182 | } 1183 | 1184 | } 1185 | 1186 | } 1187 | -------------------------------------------------------------------------------- /rst.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "errors" 29 | "github.com/xujiajun/nutsdb" 30 | "sync" 31 | ) 32 | 33 | // ResetWorking : Reset working queue 34 | func ResetWorking(cldb *nutsdb.DB, wg *sync.WaitGroup) { 35 | defer wg.Done() 36 | 37 | var err error 38 | 39 | // Wait Group 40 | 41 | wg.Add(1) 42 | 43 | // Loggers 44 | 45 | appLogger, applogfile := AppLogger() 46 | defer applogfile.Close() 47 | 48 | // Shutdown 49 | 50 | if shutdown { 51 | return 52 | } 53 | 54 | // Variables 55 | 56 | errempty := errors.New("bucket is empty") 57 | 58 | for _, Server := range config.Server { 59 | 60 | vhost := Server.HOST 61 | 62 | var rstt []ResetTask 63 | var r ResetTask 64 | 65 | wvbucket := "work" + "_" + vhost + ":" 66 | 67 | cerr := cldb.View(func(tx *nutsdb.Tx) error { 68 | 69 | var err error 70 | 71 | var tasks nutsdb.Entries 72 | 73 | tasks, err = tx.GetAll(wvbucket) 74 | 75 | if tasks == nil { 76 | return nil 77 | } 78 | 79 | if err != nil && err.Error() == errempty.Error() { 80 | return nil 81 | } 82 | 83 | if err != nil { 84 | return err 85 | } 86 | 87 | for _, rtask := range tasks { 88 | 89 | r.Key = rtask.Key 90 | rstt = append(rstt, r) 91 | 92 | } 93 | 94 | return nil 95 | 96 | }) 97 | 98 | if len(rstt) == 0 { 99 | appLogger.Warnf("| Virtual Host [%s] | Empty working queue | %v", vhost, cerr) 100 | continue 101 | } 102 | 103 | if cerr != nil { 104 | appLogger.Errorf("| Virtual Host [%s] | Work with db error | %v", vhost, cerr) 105 | continue 106 | } 107 | 108 | for _, task := range rstt { 109 | 110 | skey := string(task.Key) 111 | 112 | err = NDBDelete(cldb, wvbucket, task.Key) 113 | if err != nil { 114 | appLogger.Errorf("| Virtual Host [%s] | Initial reset task from db error | Key [%s] | %v", vhost, skey, err) 115 | } 116 | 117 | } 118 | 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /sch.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "bufio" 29 | "bytes" 30 | "encoding/gob" 31 | "errors" 32 | "fmt" 33 | "github.com/eltaline/counter" 34 | "github.com/eltaline/mmutex" 35 | "github.com/pieterclaerhout/go-waitgroup" 36 | "github.com/xujiajun/nutsdb" 37 | "math/rand" 38 | "os/exec" 39 | "regexp" 40 | "strings" 41 | "syscall" 42 | "time" 43 | ) 44 | 45 | // CtrlScheduler : Control threads scheduler 46 | func CtrlScheduler(cldb *nutsdb.DB, keymutex *mmutex.Mutex) { 47 | 48 | // Shutdown 49 | 50 | if shutdown { 51 | return 52 | } 53 | 54 | // Variables 55 | 56 | serr := errors.New("shutdown operation") 57 | kerr := errors.New("signal: killed") 58 | 59 | bprefix := []byte("t:") 60 | 61 | // Throttling 62 | 63 | vhc := 0 64 | 65 | for range config.Server { 66 | vhc++ 67 | } 68 | 69 | vwg := waitgroup.NewWaitGroup(vhc) 70 | 71 | for _, Server := range config.Server { 72 | 73 | if shutdown { 74 | continue 75 | } 76 | 77 | vwait := make(chan bool) 78 | 79 | vwg.Add(func() { 80 | 81 | vhost := Server.HOST 82 | shell := Server.SHELL 83 | 84 | rvbucket := "recv" + "_" + vhost + ":" 85 | wvbucket := "work" + "_" + vhost + ":" 86 | fvbucket := "comp" + "_" + vhost + ":" 87 | 88 | vwait <- true 89 | 90 | if keymutex.IsLock(rvbucket) { 91 | return 92 | } 93 | 94 | if keymutex.TryLock(rvbucket) { 95 | 96 | defer func() { keymutex.UnLock(rvbucket) }() 97 | 98 | mcompare := make(map[string]bool) 99 | 100 | errempty := errors.New("bucket is empty") 101 | errscans := errors.New("prefix scans not found") 102 | 103 | var ftsk []FullTask 104 | var f FullTask 105 | etsk := &RawTask{} 106 | 107 | // Loggers 108 | 109 | appLogger, applogfile := AppLogger() 110 | defer applogfile.Close() 111 | 112 | cerr := cldb.View(func(tx *nutsdb.Tx) error { 113 | 114 | var err error 115 | 116 | var received nutsdb.Entries 117 | var working nutsdb.Entries 118 | var inworking nutsdb.Entries 119 | var completed nutsdb.Entries 120 | 121 | received, _, err = tx.PrefixScan(rvbucket, bprefix, -1, -1) 122 | 123 | if received == nil { 124 | return nil 125 | } 126 | 127 | if err != nil { 128 | return err 129 | } 130 | 131 | working, err = tx.GetAll(wvbucket) 132 | 133 | if err != nil && err.Error() != errempty.Error() { 134 | return err 135 | } 136 | 137 | Main: 138 | 139 | for _, recv := range received { 140 | 141 | if shutdown { 142 | return serr 143 | } 144 | 145 | var rv RawTask 146 | 147 | rkey := recv.Key 148 | srkey := string(recv.Key) 149 | 150 | rvdec := gob.NewDecoder(bytes.NewReader(recv.Value)) 151 | err := rvdec.Decode(&rv) 152 | if err != nil { 153 | appLogger.Errorf("| Virtual Host [%s] | Gob decode from db error | %v", vhost, err) 154 | continue 155 | } 156 | 157 | srarr := strings.Split(srkey, ":") 158 | if len(srarr) == 5 { 159 | 160 | tkey := vhost + ":" + srarr[3] + ":" + srarr[4] 161 | 162 | _, found := mcompare[tkey] 163 | 164 | if found { 165 | continue 166 | } 167 | 168 | if !found { 169 | mcompare[tkey] = true 170 | } 171 | 172 | } else { 173 | 174 | appLogger.Errorf("| Virtual Host [%s] | Bad pattern in key array error | Key [%s]", vhost, srkey) 175 | continue 176 | 177 | } 178 | 179 | pair := rv.Type + "_" + rv.Lock 180 | _, found := mcompare[pair] 181 | 182 | if found { 183 | continue 184 | } 185 | 186 | if !found { 187 | mcompare[pair] = true 188 | } 189 | 190 | inworking, _, err = tx.PrefixScan(wvbucket, rkey, -1, 1) 191 | 192 | if inworking != nil { 193 | continue 194 | } 195 | 196 | if err != nil && err.Error() != errscans.Error() { 197 | return err 198 | } 199 | 200 | completed, _, err = tx.PrefixScan(fvbucket, rkey, -1, 1) 201 | 202 | if completed != nil { 203 | continue 204 | } 205 | 206 | if err != nil && err.Error() != errscans.Error() { 207 | return err 208 | } 209 | 210 | for _, work := range working { 211 | 212 | var wv RawTask 213 | 214 | wvdec := gob.NewDecoder(bytes.NewReader(work.Value)) 215 | err := wvdec.Decode(&wv) 216 | if err != nil { 217 | appLogger.Errorf("| Virtual Host [%s] | Gob decode from db error | %v", vhost, err) 218 | continue 219 | } 220 | 221 | if rv.Type == wv.Type && rv.Lock == wv.Lock { 222 | continue Main 223 | } 224 | 225 | } 226 | 227 | f.Key = recv.Key 228 | f.Time = rv.Time 229 | f.Type = rv.Type 230 | f.Path = rv.Path 231 | f.Lock = rv.Lock 232 | f.Command = rv.Command 233 | f.Threads = rv.Threads 234 | f.Timeout = rv.Timeout 235 | f.Ttltime = rv.Ttltime 236 | f.Interval = rv.Interval 237 | f.Repeaterr = rv.Repeaterr 238 | f.Repeatcnt = rv.Repeatcnt 239 | f.Interr = rv.Interr 240 | f.Intcnt = rv.Intcnt 241 | f.Lookout = rv.Lookout 242 | f.Replace = rv.Replace 243 | f.Stdcode = rv.Stdcode 244 | f.Stdout = rv.Stdout 245 | f.Errcode = rv.Errcode 246 | f.Stderr = rv.Stderr 247 | f.Runtime = rv.Runtime 248 | 249 | ftsk = append(ftsk, f) 250 | 251 | } 252 | 253 | rand.Seed(time.Now().UnixNano()) 254 | rand.Shuffle(len(ftsk), func(i, j int) { ftsk[i], ftsk[j] = ftsk[j], ftsk[i] }) 255 | 256 | /* 257 | 258 | for i := 1; i < len(ftsk); i++ { 259 | 260 | r := rand.Intn(i + 1) 261 | 262 | if i != r { 263 | ftsk[r], ftsk[i] = ftsk[i], ftsk[r] 264 | } 265 | 266 | } 267 | 268 | */ 269 | 270 | return nil 271 | 272 | }) 273 | 274 | if cerr != nil { 275 | appLogger.Errorf("| Virtual Host [%s] | Work with db error | %v", vhost, cerr) 276 | return 277 | } 278 | 279 | mpmap := make(map[string]int) 280 | 281 | qwg := waitgroup.NewWaitGroup(128) 282 | 283 | /* 284 | 285 | go func() { 286 | 287 | time.Sleep(30 * time.Second) 288 | 289 | for { 290 | 291 | if shutdown { 292 | return 293 | } 294 | 295 | c := 0 296 | 297 | for _, mp := range mpmap { 298 | c = c + mp 299 | } 300 | 301 | q := qwg.PendingCount() 302 | 303 | if c > 16 && q < 4 { 304 | keymutex.UnLock(rvbucket) 305 | break 306 | } 307 | 308 | time.Sleep(250 * time.Millisecond) 309 | 310 | } 311 | 312 | }() 313 | 314 | */ 315 | 316 | for _, task := range ftsk { 317 | 318 | if shutdown { 319 | continue 320 | } 321 | 322 | prefbkey := task.Key 323 | prefskey := string(task.Key) 324 | preftmst := time.Now().Unix() 325 | preftype := task.Type 326 | prefpath := task.Path 327 | preflock := task.Lock 328 | prefcomm := task.Command 329 | prefthreads := task.Threads 330 | preftout := task.Timeout 331 | prefttl := task.Ttltime 332 | prefint := task.Interval 333 | prefrerr := task.Repeaterr 334 | prefrcnt := task.Repeatcnt 335 | prefierr := task.Interr 336 | preficnt := task.Intcnt 337 | prefsout := task.Lookout 338 | prefrepl := task.Replace 339 | 340 | pretthr := vhost + ":" + preftype 341 | 342 | GlbMap.RLock() 343 | cntthr, ok := GlbMap.Glb[pretthr] 344 | GlbMap.RUnlock() 345 | 346 | if !ok { 347 | 348 | GlbMap.Lock() 349 | 350 | cntthr, ok = GlbMap.Glb[pretthr] 351 | if !ok { 352 | 353 | GlbMap.Glb[pretthr] = new(Cnts) 354 | cntthr = GlbMap.Glb[pretthr] 355 | cntthr.Cnt = counter.NewInt64() 356 | 357 | } 358 | 359 | GlbMap.Unlock() 360 | 361 | } 362 | 363 | if cntthr.Cnt.Get() >= int64(prefthreads) { 364 | 365 | mpc := mpmap[pretthr] 366 | 367 | if mpc > 128 { 368 | continue 369 | } 370 | 371 | mpmap[pretthr]++ 372 | 373 | } 374 | 375 | qwait := make(chan bool) 376 | 377 | qwg.Add(func() { 378 | 379 | var err error 380 | 381 | rand.Seed(time.Now().UnixNano()) 382 | 383 | intthr := pretthr 384 | 385 | GlbMap.RLock() 386 | qcntthr := GlbMap.Glb[intthr] 387 | GlbMap.RUnlock() 388 | 389 | bkey := prefbkey 390 | skey := prefskey 391 | ftmst := preftmst 392 | ftype := preftype 393 | fpath := prefpath 394 | flock := preflock 395 | fcomm := prefcomm 396 | fthreads := prefthreads 397 | ftout := preftout 398 | fttl := prefttl 399 | fint := prefint 400 | frerr := prefrerr 401 | frcnt := prefrcnt 402 | fierr := prefierr 403 | ficnt := preficnt 404 | fsout := prefsout 405 | frepl := prefrepl 406 | 407 | qwait <- true 408 | 409 | if qcntthr.Cnt.Get() >= int64(fthreads) { 410 | 411 | for { 412 | 413 | if qcntthr.Cnt.Get() < int64(fthreads) { 414 | break 415 | } 416 | 417 | time.Sleep(time.Duration(rand.Intn(250)) * time.Millisecond) 418 | 419 | } 420 | 421 | } 422 | 423 | qcntthr.Cnt.Incr() 424 | 425 | vtscnt := qcntthr.Cnt.Get() 426 | 427 | stdcode := 0 428 | errcode := 0 429 | 430 | stdout := "" 431 | stderr := "" 432 | 433 | lookout := fsout 434 | 435 | var ig Ereg 436 | var rg Ereg 437 | 438 | var ireg []Ereg 439 | var rreg []Ereg 440 | 441 | for _, rgxint := range fierr { 442 | 443 | irgx, err := regexp.Compile(rgxint) 444 | if err != nil { 445 | appLogger.Errorf("| Virtual Host [%s] | Bad pattern in intercept errors array | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | Regular [%s] | %v", vhost, skey, fpath, flock, fcomm, rgxint, err) 446 | continue 447 | } 448 | 449 | ig.Str = rgxint 450 | ig.Rgx = irgx 451 | 452 | ireg = append(ireg, ig) 453 | 454 | } 455 | 456 | for _, rgxrpt := range frerr { 457 | 458 | rrgx, err := regexp.Compile(rgxrpt) 459 | if err != nil { 460 | appLogger.Errorf("| Virtual Host [%s] | Bad pattern in repeat errors array | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | Regular [%s] | %v", vhost, skey, fpath, flock, fcomm, rgxrpt, err) 461 | continue 462 | } 463 | 464 | rg.Str = rgxrpt 465 | rg.Rgx = rrgx 466 | 467 | rreg = append(rreg, rg) 468 | 469 | } 470 | 471 | lenireg := len(ireg) 472 | lenrreg := len(rreg) 473 | 474 | pbuffer := new(bytes.Buffer) 475 | penc := gob.NewEncoder(pbuffer) 476 | 477 | if !DirExists(fpath) { 478 | 479 | stderr = fmt.Sprintf("Path [%s] | directory from json field path does not exists", fpath) 480 | 481 | errcode = 1 482 | 483 | appLogger.Errorf("| Virtual Host [%s] | Can`t find directory error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 484 | 485 | etsk = &RawTask{ 486 | Time: ftmst, 487 | Type: ftype, 488 | Path: fpath, 489 | Lock: flock, 490 | Command: fcomm, 491 | Threads: fthreads, 492 | Timeout: ftout, 493 | Ttltime: fttl, 494 | Interval: fint, 495 | Repeaterr: frerr, 496 | Repeatcnt: frcnt, 497 | Interr: fierr, 498 | Intcnt: ficnt, 499 | Lookout: fsout, 500 | Replace: frepl, 501 | Stdcode: stdcode, 502 | Stdout: stdout, 503 | Errcode: errcode, 504 | Stderr: stderr, 505 | Runtime: float64(0), 506 | } 507 | 508 | err = penc.Encode(etsk) 509 | if err != nil { 510 | qcntthr.Cnt.Decr() 511 | appLogger.Errorf("| Virtual Host [%s] | Gob task encode error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 512 | return 513 | } 514 | 515 | err = NDBDelete(cldb, rvbucket, bkey) 516 | if err != nil { 517 | qcntthr.Cnt.Decr() 518 | appLogger.Errorf("| Virtual Host [%s] | Delete received task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 519 | return 520 | } 521 | 522 | err = NDBInsert(cldb, fvbucket, bkey, pbuffer.Bytes(), 0) 523 | if err != nil { 524 | qcntthr.Cnt.Decr() 525 | appLogger.Errorf("| Virtual Host [%s] | Insert completed task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 526 | return 527 | } 528 | 529 | qcntthr.Cnt.Decr() 530 | return 531 | 532 | } 533 | 534 | scm := shell + " -c " + "\"cd " + fpath + " && " + fcomm + "\"" 535 | cmm := exec.Command(shell, "-c", scm) 536 | cmm.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} 537 | 538 | pstdout, err := cmm.StdoutPipe() 539 | if err != nil { 540 | qcntthr.Cnt.Decr() 541 | appLogger.Errorf("| Virtual Host [%s] | Can`t attach to stdout pipe | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 542 | return 543 | } 544 | 545 | pstderr, err := cmm.StderrPipe() 546 | if err != nil { 547 | qcntthr.Cnt.Decr() 548 | appLogger.Errorf("| Virtual Host [%s] | Can`t attach to stderr pipe | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 549 | return 550 | } 551 | 552 | etsk = &RawTask{ 553 | Time: ftmst, 554 | Type: ftype, 555 | Path: fpath, 556 | Lock: flock, 557 | Command: fcomm, 558 | Threads: fthreads, 559 | Timeout: ftout, 560 | Ttltime: fttl, 561 | Interval: fint, 562 | Repeaterr: frerr, 563 | Repeatcnt: frcnt, 564 | Interr: fierr, 565 | Intcnt: ficnt, 566 | Lookout: fsout, 567 | Replace: frepl, 568 | Stdcode: stdcode, 569 | Stdout: stdout, 570 | Errcode: errcode, 571 | Stderr: "", 572 | Runtime: float64(0), 573 | } 574 | 575 | err = penc.Encode(etsk) 576 | if err != nil { 577 | qcntthr.Cnt.Decr() 578 | appLogger.Errorf("| Virtual Host [%s] | Gob task encode error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 579 | return 580 | } 581 | 582 | err = NDBInsert(cldb, wvbucket, bkey, pbuffer.Bytes(), 0) 583 | if err != nil { 584 | qcntthr.Cnt.Decr() 585 | appLogger.Errorf("| Virtual Host [%s] | Insert working task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 586 | return 587 | } 588 | 589 | var rtime float64 590 | 591 | cwg := waitgroup.NewWaitGroup(1) 592 | 593 | var intercept bool 594 | 595 | imsgout := "" 596 | imsgerr := "" 597 | 598 | crun := make(chan bool) 599 | kill := make(chan bool) 600 | kwait := make(chan bool) 601 | owait := make(chan bool) 602 | ewait := make(chan bool) 603 | 604 | ssync := make(chan bool) 605 | 606 | cmkey := vhost + ":" + skey 607 | 608 | stime := time.Now() 609 | 610 | time.Sleep(time.Duration(int(fint)*(int(vtscnt)-1)) * time.Second) 611 | 612 | killed := false 613 | 614 | if keymutex.TryLock(cmkey) { 615 | 616 | cwg.Add(func() { 617 | 618 | <-ssync 619 | close(ssync) 620 | 621 | var err error 622 | 623 | err = cmm.Start() 624 | if err != nil { 625 | 626 | if exitError, ok := err.(*exec.ExitError); ok { 627 | errcode = exitError.ExitCode() 628 | } else { 629 | errcode = 255 630 | } 631 | 632 | imsgerr = err.Error() 633 | appLogger.Errorf("| Virtual Host [%s] | Start command error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, scm, err) 634 | 635 | } 636 | 637 | err = cmm.Wait() 638 | if err != nil { 639 | 640 | if exitError, ok := err.(*exec.ExitError); ok { 641 | errcode = exitError.ExitCode() 642 | } else { 643 | errcode = 1 644 | } 645 | 646 | if !shutdown { 647 | appLogger.Errorf("| Virtual Host [%s] | Execute command error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, scm, err) 648 | } 649 | 650 | if err.Error() == kerr.Error() { 651 | 652 | err = NDBDelete(cldb, wvbucket, bkey) 653 | if err != nil { 654 | appLogger.Errorf("| Virtual Host [%s] | Delete working task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 655 | } 656 | 657 | } 658 | 659 | } 660 | 661 | if !killed { 662 | crun <- true 663 | } 664 | 665 | }) 666 | 667 | } else { 668 | 669 | errcode = 255 670 | imsgerr = "Timeout mmutex lock error" 671 | appLogger.Errorf("| Virtual Host [%s] | Timeout mmutex lock error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | Error [%s] | %v", vhost, skey, fpath, flock, scm, imsgerr, err) 672 | 673 | err = NDBDelete(cldb, wvbucket, bkey) 674 | if err != nil { 675 | qcntthr.Cnt.Decr() 676 | appLogger.Errorf("| Virtual Host [%s] | Delete working task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 677 | return 678 | } 679 | 680 | qcntthr.Cnt.Decr() 681 | return 682 | 683 | } 684 | 685 | kwg := waitgroup.NewWaitGroup(1) 686 | 687 | kwg.Add(func() { 688 | 689 | kwait <- true 690 | 691 | for { 692 | 693 | if !keymutex.IsLock(cmkey) || killed || shutdown { 694 | 695 | if !killed { 696 | kill <- true 697 | } 698 | 699 | break 700 | 701 | } 702 | 703 | time.Sleep(time.Duration(5) * time.Millisecond) 704 | 705 | } 706 | 707 | }) 708 | 709 | <-kwait 710 | close(kwait) 711 | 712 | owg := waitgroup.NewWaitGroup(1) 713 | 714 | owg.Add(func() { 715 | 716 | scanner := bufio.NewScanner(pstdout) 717 | 718 | owait <- true 719 | 720 | Loop: 721 | 722 | for scanner.Scan() { 723 | 724 | msg := scanner.Text() 725 | 726 | imsgout = imsgout + msg + "\n" 727 | 728 | if lookout && lenireg > 0 { 729 | 730 | for _, rgx := range ireg { 731 | 732 | mchvir := rgx.Rgx.MatchString(msg) 733 | 734 | if mchvir && keymutex.IsLock(cmkey) && !killed { 735 | 736 | intercept = true 737 | kill <- true 738 | break Loop 739 | 740 | } 741 | 742 | if killed || shutdown { 743 | break Loop 744 | } 745 | 746 | } 747 | 748 | } 749 | 750 | if !keymutex.IsLock(cmkey) || killed || shutdown { 751 | break 752 | } 753 | 754 | } 755 | 756 | }) 757 | 758 | <-owait 759 | close(owait) 760 | 761 | ewg := waitgroup.NewWaitGroup(1) 762 | 763 | ewg.Add(func() { 764 | 765 | scanner := bufio.NewScanner(pstderr) 766 | 767 | ewait <- true 768 | 769 | Loop: 770 | 771 | for scanner.Scan() { 772 | 773 | msg := scanner.Text() 774 | 775 | imsgerr = imsgerr + msg + "\n" 776 | 777 | if lenireg > 0 { 778 | 779 | for _, rgx := range ireg { 780 | 781 | mchvir := rgx.Rgx.MatchString(msg) 782 | 783 | if mchvir && keymutex.IsLock(cmkey) && !killed { 784 | 785 | intercept = true 786 | kill <- true 787 | break Loop 788 | 789 | } 790 | 791 | if killed || shutdown { 792 | break Loop 793 | } 794 | 795 | } 796 | 797 | } 798 | 799 | if !keymutex.IsLock(cmkey) || killed || shutdown { 800 | break 801 | } 802 | 803 | } 804 | 805 | }) 806 | 807 | <-ewait 808 | close(ewait) 809 | 810 | ssync <- true 811 | 812 | cmmt := time.After(time.Duration(int(ftout)) * time.Second) 813 | 814 | Kill: 815 | 816 | for { 817 | 818 | select { 819 | 820 | case <-crun: 821 | 822 | killed = true 823 | break Kill 824 | 825 | case <-kill: 826 | 827 | if cmm.Process != nil { 828 | 829 | pgid, err := syscall.Getpgid(cmm.Process.Pid) 830 | if err == nil { 831 | 832 | zerr := syscall.Kill(-pgid, syscall.SIGKILL) 833 | if zerr != nil { 834 | appLogger.Errorf("| Virtual Host [%s] | Process kill error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, zerr) 835 | } 836 | 837 | } 838 | 839 | } 840 | 841 | killed = true 842 | break Kill 843 | 844 | case <-cmmt: 845 | 846 | if cmm.Process != nil { 847 | 848 | pgid, err := syscall.Getpgid(cmm.Process.Pid) 849 | if err == nil { 850 | 851 | zerr := syscall.Kill(-pgid, syscall.SIGKILL) 852 | if zerr != nil { 853 | appLogger.Errorf("| Virtual Host [%s] | Process kill error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, zerr) 854 | } 855 | 856 | } 857 | 858 | } 859 | 860 | killed = true 861 | errcode = 124 862 | break Kill 863 | 864 | default: 865 | 866 | time.Sleep(time.Duration(5) * time.Millisecond) 867 | 868 | } 869 | 870 | } 871 | 872 | cwg.Wait() 873 | close(crun) 874 | 875 | if keymutex.IsLock(cmkey) { 876 | keymutex.UnLock(cmkey) 877 | } 878 | 879 | kwg.Wait() 880 | owg.Wait() 881 | ewg.Wait() 882 | close(kill) 883 | 884 | if shutdown { 885 | qcntthr.Cnt.Decr() 886 | return 887 | } 888 | 889 | stdout = imsgout 890 | stderr = imsgerr 891 | 892 | rtime = float64(time.Since(stime)) / float64(time.Millisecond) 893 | 894 | vircnt := 0 895 | virbool := false 896 | 897 | vrrcnt := 0 898 | vrrbool := false 899 | 900 | if intercept && lenireg > 0 { 901 | 902 | iskey := "i:" + skey 903 | 904 | GlbMap.RLock() 905 | cnticnt, ok := GlbMap.Glb[iskey] 906 | GlbMap.RUnlock() 907 | 908 | if !ok { 909 | 910 | GlbMap.Lock() 911 | 912 | cnticnt, ok = GlbMap.Glb[iskey] 913 | if !ok { 914 | 915 | GlbMap.Glb[iskey] = new(Cnts) 916 | cnticnt = GlbMap.Glb[iskey] 917 | cnticnt.Cnt = counter.NewInt64() 918 | 919 | } 920 | 921 | GlbMap.Unlock() 922 | 923 | } 924 | 925 | for _, rgx := range ireg { 926 | 927 | var mchvio bool 928 | 929 | mchvir := rgx.Rgx.MatchString(stderr) 930 | 931 | if lookout { 932 | mchvio = rgx.Rgx.MatchString(stdout) 933 | } 934 | 935 | if mchvir || mchvio { 936 | 937 | if vircnt == 0 { 938 | 939 | virbool = true 940 | 941 | cnticnt.Cnt.Incr() 942 | 943 | vircnt++ 944 | 945 | } 946 | 947 | } 948 | 949 | } 950 | 951 | virc := cnticnt.Cnt.Get() 952 | 953 | if virbool && int(virc) > 0 && int(virc) <= int(ficnt) { 954 | 955 | err = NDBDelete(cldb, wvbucket, bkey) 956 | if err != nil { 957 | qcntthr.Cnt.Decr() 958 | appLogger.Errorf("| Virtual Host [%s] | Delete working task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 959 | return 960 | } 961 | 962 | qcntthr.Cnt.Decr() 963 | return 964 | 965 | } 966 | 967 | GlbMap.Lock() 968 | _, ok = GlbMap.Glb[iskey] 969 | if ok { 970 | delete(GlbMap.Glb, iskey) 971 | } 972 | GlbMap.Unlock() 973 | 974 | } 975 | 976 | if lenrreg > 0 { 977 | 978 | rskey := "r:" + skey 979 | 980 | GlbMap.RLock() 981 | cntrcnt, ok := GlbMap.Glb[rskey] 982 | GlbMap.RUnlock() 983 | 984 | if !ok { 985 | 986 | GlbMap.Lock() 987 | 988 | cntrcnt, ok = GlbMap.Glb[rskey] 989 | if !ok { 990 | 991 | GlbMap.Glb[rskey] = new(Cnts) 992 | cntrcnt = GlbMap.Glb[rskey] 993 | cntrcnt.Cnt = counter.NewInt64() 994 | 995 | } 996 | 997 | GlbMap.Unlock() 998 | 999 | } 1000 | 1001 | for _, rgx := range rreg { 1002 | 1003 | var mchvoo bool 1004 | 1005 | mchvrr := rgx.Rgx.MatchString(stderr) 1006 | 1007 | if lookout { 1008 | mchvoo = rgx.Rgx.MatchString(stdout) 1009 | } 1010 | 1011 | if mchvrr || mchvoo { 1012 | 1013 | if vrrcnt == 0 { 1014 | 1015 | vrrbool = true 1016 | 1017 | cntrcnt.Cnt.Incr() 1018 | 1019 | vrrcnt++ 1020 | 1021 | } 1022 | 1023 | } 1024 | 1025 | } 1026 | 1027 | vrrc := cntrcnt.Cnt.Incr() 1028 | 1029 | if vrrbool && int(vrrc) > 0 && int(vrrc) <= int(frcnt) { 1030 | 1031 | err = NDBDelete(cldb, wvbucket, bkey) 1032 | if err != nil { 1033 | qcntthr.Cnt.Decr() 1034 | appLogger.Errorf("| Virtual Host [%s] | Delete working task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 1035 | return 1036 | } 1037 | 1038 | qcntthr.Cnt.Decr() 1039 | return 1040 | 1041 | } 1042 | 1043 | GlbMap.Lock() 1044 | _, ok = GlbMap.Glb[rskey] 1045 | if ok { 1046 | delete(GlbMap.Glb, rskey) 1047 | } 1048 | GlbMap.Unlock() 1049 | 1050 | } 1051 | 1052 | ebuffer := new(bytes.Buffer) 1053 | eenc := gob.NewEncoder(ebuffer) 1054 | 1055 | ftmst = time.Now().Unix() 1056 | 1057 | etsk = &RawTask{ 1058 | Time: ftmst, 1059 | Type: ftype, 1060 | Path: fpath, 1061 | Lock: flock, 1062 | Command: fcomm, 1063 | Threads: fthreads, 1064 | Timeout: ftout, 1065 | Ttltime: fttl, 1066 | Interval: fint, 1067 | Repeaterr: frerr, 1068 | Repeatcnt: frcnt, 1069 | Interr: fierr, 1070 | Intcnt: ficnt, 1071 | Lookout: fsout, 1072 | Replace: frepl, 1073 | Stdcode: stdcode, 1074 | Stdout: stdout, 1075 | Errcode: errcode, 1076 | Stderr: stderr, 1077 | Runtime: rtime, 1078 | } 1079 | 1080 | err = eenc.Encode(etsk) 1081 | if err != nil { 1082 | appLogger.Errorf("| Virtual Host [%s] | Gob task encode error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 1083 | qcntthr.Cnt.Decr() 1084 | return 1085 | } 1086 | 1087 | err = NDBDelete(cldb, rvbucket, bkey) 1088 | if err != nil { 1089 | appLogger.Errorf("| Virtual Host [%s] | Delete received task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 1090 | qcntthr.Cnt.Decr() 1091 | return 1092 | } 1093 | 1094 | err = NDBDelete(cldb, wvbucket, bkey) 1095 | if err != nil { 1096 | appLogger.Errorf("| Virtual Host [%s] | Delete working task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 1097 | qcntthr.Cnt.Decr() 1098 | return 1099 | } 1100 | 1101 | err = NDBInsert(cldb, fvbucket, bkey, ebuffer.Bytes(), fttl) 1102 | if err != nil { 1103 | appLogger.Errorf("| Virtual Host [%s] | Insert completed task db error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | %v", vhost, skey, fpath, flock, fcomm, err) 1104 | qcntthr.Cnt.Decr() 1105 | return 1106 | } 1107 | 1108 | if errcode != 0 { 1109 | appLogger.Errorf("| Virtual Host [%s] | Task completed with error | Key [%s] | Path [%s] | Lock [%s] | Command [%s] | Error [%s] | exit status %d", vhost, skey, fpath, flock, fcomm, stderr, errcode) 1110 | } 1111 | 1112 | qcntthr.Cnt.Decr() 1113 | 1114 | }) 1115 | 1116 | <-qwait 1117 | close(qwait) 1118 | 1119 | time.Sleep(250 * time.Millisecond) 1120 | 1121 | } 1122 | 1123 | qwg.Wait() 1124 | 1125 | } 1126 | 1127 | }) 1128 | 1129 | <-vwait 1130 | close(vwait) 1131 | 1132 | } 1133 | 1134 | vwg.Wait() 1135 | 1136 | } 1137 | -------------------------------------------------------------------------------- /scripts/postinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -z `getent group ctrl` ]; then 4 | groupadd ctrl 5 | fi 6 | 7 | if [ -z `getent passwd ctrl` ]; then 8 | useradd ctrl -g ctrl -s /bin/sh 9 | fi 10 | 11 | if [ ! -d /var/lib/ctrl ] ; then 12 | install --mode=755 --owner=ctrl --group=ctrl --directory /var/lib/ctrl 13 | fi 14 | 15 | if [ ! -d /var/log/ctrl ] ; then 16 | install --mode=755 --owner=ctrl --group=ctrl --directory /var/log/ctrl 17 | fi 18 | 19 | systemctl daemon-reload 20 | 21 | #END -------------------------------------------------------------------------------- /sys.go: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | Copyright © 2020 Andrey Kuvshinov. Contacts: 4 | Copyright © 2020 Eltaline OU. Contacts: 5 | All rights reserved. 6 | 7 | Licensed under the Apache License, Version 2.0 (the "License"); 8 | you may not use this file except in compliance with the License. 9 | You may obtain a copy of the License at 10 | 11 | http://www.apache.org/licenses/LICENSE-2.0 12 | 13 | Unless required by applicable law or agreed to in writing, software 14 | distributed under the License is distributed on an "AS IS" BASIS, 15 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | See the License for the specific language governing permissions and 17 | limitations under the License. 18 | 19 | The cTRL project contains unmodified/modified libraries imports too with 20 | separate copyright notices and license terms. Your use of the source code 21 | this libraries is subject to the terms and conditions of licenses these libraries. 22 | 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "fmt" 29 | "os" 30 | ) 31 | 32 | // System Helpers 33 | 34 | // GetPID : get current pid number and return int and string representation of pid 35 | func GetPID() (gpid string, fpid string) { 36 | 37 | gpid = fmt.Sprintf("%d", os.Getpid()) 38 | fpid = fmt.Sprintf("%s\n", gpid) 39 | 40 | return gpid, fpid 41 | 42 | } 43 | --------------------------------------------------------------------------------