├── .dockerignore ├── .github ├── FUNDING.yml └── workflows │ └── go.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── Dockerfile ├── Dockerfile.build ├── LICENSE ├── README.md ├── _docs ├── api.md ├── authorization.md ├── cron.md ├── docker.md ├── flow.md ├── flow.txt ├── goals.md ├── internals │ └── worker_flow ├── modes.md ├── nano-run.svg ├── ui.md ├── ui_authorization.md └── unit.md ├── _footer.md ├── bundle ├── debian │ ├── nano-run.service │ ├── postinstall.sh │ ├── preremove.sh │ └── server.yaml └── docker │ └── server.yaml ├── cmd └── nano-run │ ├── main.go │ ├── run_cmd.go │ ├── server_cmd.go │ ├── signals_default.go │ └── signals_posix.go ├── go.mod ├── go.sum ├── internal └── nano_logger.go ├── server ├── api │ └── adapter.go ├── auth.go ├── cron.go ├── internal │ ├── flags.go │ └── flags_linux.go ├── mode_bin.go ├── mode_bin_default.go ├── mode_bin_linux.go ├── runner │ ├── embedded │ │ ├── static │ │ │ └── bindata.go │ │ └── templates │ │ │ └── bindata.go │ └── handler.go ├── server_test.go ├── ui │ ├── authorization.go │ ├── oauth.go │ ├── router.go │ ├── sessions.go │ ├── unit_info.go │ └── utils.go └── unit.go ├── services ├── blob │ ├── fsblob │ │ └── impl.go │ └── interface.go ├── meta │ ├── interface.go │ └── micrometa │ │ └── request_meta_storage.go └── queue │ ├── interface.go │ └── microqueue │ └── micro_queue.go ├── templates ├── cron-list.html ├── login.html ├── static │ └── css │ │ └── bootstrap-material-design.min.css ├── unit-cron-info.html ├── unit-info.html ├── unit-request-attempt-info.html ├── unit-request-info.html └── units-list.html └── worker ├── doc.go ├── request_io.go ├── tracker.go └── worker.go /.dockerignore: -------------------------------------------------------------------------------- 1 | Dockerfile 2 | /.idea 3 | /_docs 4 | /build 5 | /dist 6 | /run 7 | /conf.d -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] 4 | patreon: # Replace with a single Patreon username 5 | open_collective: # Replace with a single Open Collective username 6 | ko_fi: # Replace with a single Ko-fi username 7 | tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel 8 | community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry 9 | liberapay: # Replace with a single Liberapay username 10 | issuehunt: # Replace with a single IssueHunt username 11 | otechie: # Replace with a single Otechie username 12 | custom: ['http://reddec.net/about/#donate'] 13 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Build project 2 | on: 3 | push: 4 | tags: 5 | - 'v*' 6 | jobs: 7 | 8 | build: 9 | name: Build 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Set up Go 1.15 13 | uses: actions/setup-go@v1 14 | with: 15 | go-version: '^1.15' 16 | id: go 17 | 18 | - name: Check out code into the Go module directory 19 | uses: actions/checkout@v2 20 | with: 21 | lfs: true 22 | fetch-depth: 0 23 | - name: Checkout LFS objects 24 | run: git lfs checkout 25 | 26 | - name: Pull tag 27 | run: git fetch --tags 28 | - uses: azure/docker-login@v1 29 | with: 30 | username: 'reddec' 31 | password: ${{ secrets.DOCKERIO_PASSWORD }} 32 | 33 | - name: Run GoReleaser 34 | uses: goreleaser/goreleaser-action@v2 35 | with: 36 | version: latest 37 | args: release --rm-dist --release-footer _footer.md 38 | env: 39 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | UPLOAD_BINTRAY_SECRET: ${{ secrets.UPLOAD_BINTRAY_SECRET }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /run 3 | /dist 4 | /build -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | tests: false 3 | linters: 4 | disable-all: false 5 | enable: 6 | - bodyclose 7 | - deadcode 8 | - depguard 9 | - dogsled 10 | - dupl 11 | - errcheck 12 | - exhaustive 13 | - funlen 14 | - gochecknoinits 15 | - goconst 16 | - gocyclo 17 | - gofmt 18 | - goimports 19 | - golint 20 | - gomnd 21 | - goprintffuncname 22 | - gosec 23 | - gosimple 24 | - govet 25 | - ineffassign 26 | - interfacer 27 | - misspell 28 | - nakedret 29 | - noctx 30 | - nolintlint 31 | - rowserrcheck 32 | - scopelint 33 | - staticcheck 34 | - structcheck 35 | - stylecheck 36 | - typecheck 37 | - unconvert 38 | - unparam 39 | - unused 40 | - varcheck 41 | - whitespace -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | project_name: nano-run 2 | builds: 3 | - env: 4 | - CGO_ENABLED=0 5 | goos: 6 | - linux 7 | - windows 8 | - darwin 9 | goarch: 10 | - amd64 11 | - arm64 12 | - arm 13 | goarm: 14 | - 5 15 | - 6 16 | - 7 17 | flags: 18 | - -trimpath 19 | main: ./cmd/nano-run 20 | 21 | nfpms: 22 | - id: debian 23 | file_name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}" 24 | replacements: 25 | Linux: linux 26 | 386: i386 27 | homepage: https://github.com/reddec/nano-run 28 | maintainer: Baryshnikov Aleksandr 29 | description: Lightweigt runner for web requests 30 | license: Apache-2.0 31 | formats: 32 | - deb 33 | scripts: 34 | postinstall: "bundle/debian/postinstall.sh" 35 | preremove: "bundle/debian/preremove.sh" 36 | empty_folders: 37 | - /etc/nano-run/conf.d 38 | - /var/nano-run 39 | config_files: 40 | "bundle/debian/server.yaml": "/etc/nano-run/server.yaml" 41 | files: 42 | "bundle/debian/nano-run.service": "/etc/systemd/system/nano-run.service" 43 | "templates/**/*": "/var/nano-run/ui" 44 | archives: 45 | - files: 46 | - "templates/**/*" 47 | wrap_in_directory: true 48 | 49 | uploads: 50 | - name: bintray 51 | method: PUT 52 | mode: archive 53 | username: reddec 54 | custom_artifact_name: true 55 | ids: 56 | - debian 57 | target: 'https://api.bintray.com/content/reddec/debian/{{ .ProjectName }}/{{ .Version }}/{{ .ArtifactName }};publish=1;deb_component=main;deb_distribution=all;deb_architecture={{ .Arch }}' 58 | dockers: 59 | - binaries: 60 | - nano-run 61 | dockerfile: Dockerfile 62 | extra_files: 63 | - bundle/docker/server.yaml 64 | - templates 65 | build_flag_templates: 66 | - "--label=org.opencontainers.image.created={{.Date}}" 67 | - "--label=org.opencontainers.image.title={{.ProjectName}}" 68 | - "--label=org.opencontainers.image.revision={{.FullCommit}}" 69 | - "--label=org.opencontainers.image.version={{.Version}}" 70 | image_templates: 71 | - "reddec/nano-run:{{ .Tag }}" 72 | - "reddec/nano-run:v{{ .Major }}" 73 | - "reddec/nano-run:v{{ .Major }}.{{ .Minor }}" 74 | - "reddec/nano-run:latest" 75 | checksum: 76 | name_template: 'checksums.txt' 77 | snapshot: 78 | name_template: "{{ .Tag }}-next" 79 | changelog: 80 | sort: asc 81 | filters: 82 | exclude: 83 | - '^docs:' 84 | - '^test:' 85 | - '^build:' 86 | - '^lint:' 87 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.12 2 | ENV GIN_MODE=release 3 | VOLUME /data 4 | VOLUME /conf.d 5 | EXPOSE 80 6 | COPY templates /ui 7 | COPY nano-run /bin/nano-run 8 | COPY bundle/docker/server.yaml /server.yaml 9 | CMD ["/bin/nano-run", "server", "run", "-f", "-c", "server.yaml"] -------------------------------------------------------------------------------- /Dockerfile.build: -------------------------------------------------------------------------------- 1 | FROM golang:1.15-alpine3.12 AS build 2 | WORKDIR /go/src/app 3 | COPY . . 4 | 5 | RUN go get -d -v ./... 6 | RUN go build -o nano-run -v ./cmd/nano-run/... 7 | 8 | FROM alpine:3.12 9 | VOLUME /data 10 | VOLUME /conf.d 11 | COPY docker/server.yaml /server.yaml 12 | COPY --from=build /go/src/app/nano-run /bin/nano-run 13 | CMD ["/bin/nano-run", "server", "run", "-f", "-c", "server.yaml"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Nano-Run 2 | 3 | ![nano-run](https://user-images.githubusercontent.com/6597086/97143787-4f70db80-179e-11eb-9b9c-7e16bfff845e.png) 4 | 5 | Lightweight async request runner. GitOps friendly. 6 | 7 | A simplified version of [trusted-cgi](https://github.com/reddec/trusted-cgi) designed 8 | for async processing extreme amount of requests. 9 | 10 | Main goals: 11 | 12 | * Should have semi-constant resource consumption regardless of: 13 | * number of requests, 14 | * size of requests, 15 | * kind of requests; 16 | * Should be ready to run without configuration; 17 | * Should be ready for deploying in clouds; 18 | * Should support extending for another providers; 19 | * Can be used as library and as a complete solution; 20 | * **Performance (throughput/latency) has less priority** than resource usage. 21 | 22 | Please note that the project is being developed in free time, as a non-profitable hobby project. 23 | All codes, bugs, opinions, and other related subjects should not be considered as the official position, official project, 24 | or company-backed project to any of the companies for/with which I worked before or/and at present. 25 | 26 | TODO: 27 | 28 | * On attempt failed job 29 | * On failed job 30 | * On success job 31 | * Per endpoint swagger 32 | * Pickup file as an artifact 33 | 34 | 35 | ![sample-nano-run](https://user-images.githubusercontent.com/6597086/98463432-303e6900-21f6-11eb-9632-806b1c99813b.gif) 36 | 37 | ## Documentation 38 | 39 | * [Installation](#installation) 40 | * [Quick start](#quick-start) 41 | * [Architecture overview](_docs/flow.md) 42 | * [API](_docs/api.md) 43 | * [API Authorization](_docs/authorization.md) 44 | * [UI](_docs/ui.md) 45 | * [UI Authorization](_docs/ui_authorization.md) 46 | * [Unit configuration](_docs/unit.md) 47 | * [Docker](_docs/docker.md) 48 | * [Cron-like scheduler](_docs/cron.md) 49 | 50 | ## Stability 51 | 52 | (After 1.0.0) 53 | 54 | We are trying to follow semver: 55 | 56 | * Patch releases provides fixes or light improvements without migration 57 | * Minor releases provides new functionality and provides automatic migration if needed 58 | * Major releases provides serious platform changes and may require manual or automatic migration 59 | 60 | Within one major release, it guarantees forward compatibility: new versions can use data from previous versions, but not vice-versa. 61 | 62 | ## Reproducible binaries 63 | 64 | The project tries to follow best practices providing reproducible binaries: it means, that 65 | you can verify that complied binaries will be exactly the same (byte to byte) as if you will compile it by yourself 66 | by following our public instructions. 67 | 68 | ## License 69 | 70 | The project (source code and provided official binaries) are licensed 71 | under Apache-2.0 (see License file, or in [plain English](https://tldrlegal.com/license/apache-license-2.0-(apache-2.0))) and suitable 72 | for personal, commercial, government, and others usage without restrictions as long as it used with abiding 73 | license agreement. 74 | 75 | Do not forget to bring your changes back to the project. I will 76 | be happy to assist you with PR. 77 | 78 | The project uses external libraries that may be distributed 79 | under other licenses. 80 | 81 | ## Installation 82 | 83 | ### Debian/Ubuntu 84 | 85 | (recommended) 86 | 87 | Tested on 20.04 and 18.04, but should good for any x64 version. 88 | 89 | Add the repository (only once) 90 | 91 | ```bash 92 | sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 93 | echo "deb https://dl.bintray.com/reddec/debian all main" | sudo tee /etc/apt/sources.list.d/reddec.list 94 | ``` 95 | 96 | Install or update nano-run 97 | 98 | ```bash 99 | sudo apt update 100 | sudo apt install nano-run 101 | ``` 102 | 103 | Automatically creates service `nano-run.service`. 104 | 105 | ### Binary 106 | 107 | Download and unpack desired version in [releases](https://github.com/reddec/nano-run/releases). 108 | 109 | ### Docker 110 | 111 | `docker pull reddec/nano-run` 112 | 113 | ### From source 114 | 115 | Requires go 1.14+ 116 | 117 | `go get -v github.com/reddec/nano-run/cmd/...` 118 | 119 | ### Ansible for debian servers 120 | 121 | 122 | ```yaml 123 | - name: Add an apt key by id from a keyserver 124 | become: yes 125 | apt_key: 126 | keyserver: keyserver.ubuntu.com 127 | id: 379CE192D401AB61 128 | - name: Add repository 129 | become: yes 130 | apt_repository: 131 | repo: deb https://dl.bintray.com/reddec/debian all main 132 | state: present 133 | filename: reddec 134 | - name: Install nano-run 135 | become: yes 136 | apt: 137 | name: nano-run 138 | update_cache: yes 139 | state: latest 140 | ``` 141 | 142 | ## Quick start 143 | 144 | **(optional) initialize configuration** 145 | 146 | nano-run server init 147 | 148 | it will create required directories and files in a current working directory. 149 | 150 | **define a unit file** 151 | 152 | Create minimal unit file (date.yaml) that will return current date (by unix command `date`) and put it 153 | in directory `run/conf.d/` 154 | 155 | _run/conf.d/date.yaml_ 156 | ```yaml 157 | command: date 158 | ``` 159 | 160 | **start nano-run** 161 | 162 | nano-run server run 163 | 164 | 165 | now you can open ui over http://localhost:8989 or do API call: `curl -X POST http://localhost:8989/api/date/` 166 | -------------------------------------------------------------------------------- /_docs/api.md: -------------------------------------------------------------------------------- 1 | # API 2 | 3 | Base url: `/{application}` 4 | 5 | 6 | ## Start process 7 | 8 | ||| 9 | |------------|------| 10 | | **Method** | POST | 11 | | **Path** | / | 12 | 13 | Saves and enqueue request. Return 303 See Other on success with headers. 14 | 15 | * `X-Correlation-Id` - with ID of request 16 | * `Location` - URL to status 17 | 18 | Clients can use cUrl flag `-L` to automatically follow redirect 19 | 20 | curl -L -d '' 'http://127.0.0.1:8989/app/' 21 | 22 | ## Get status 23 | 24 | 25 | ||| 26 | |------------|------------------| 27 | | **Method** | GET | 28 | | **Path** | /{correlationId} | 29 | 30 | Returns JSON with full meta-data of request and following headers: 31 | 32 | * `Content-Version` - number of attempts 33 | * `Last-Modified` - latest of time of creation, time of last attempt or completion time 34 | * `Location` - URL to the complete attempt 35 | * `X-Status` - status of request processing: `complete` or `processing` 36 | * `X-Last-Attempt` - id of last attempt 37 | * `X-Last-Attempt-At` - time of last attempt 38 | * `X-Correlation-Id` - with ID of request 39 | 40 | Body example: 41 | 42 | ```json 43 | { 44 | "created_at": "2020-09-10T17:11:33.598542177+08:00", 45 | "complete_at": "2020-09-10T17:11:33.616550544+08:00", 46 | "attempts": [ 47 | { 48 | "code": 200, 49 | "headers": {}, 50 | "id": "51748767-e89b-48a1-8b00-b9c1f0fdc9bb", 51 | "created_at": "2020-09-10T17:11:33.614030236+08:00" 52 | } 53 | ], 54 | "headers": { 55 | "Accept": [ 56 | "*/*" 57 | ], 58 | "Content-Length": [ 59 | "0" 60 | ], 61 | "Content-Type": [ 62 | "application/x-www-form-urlencoded" 63 | ], 64 | "User-Agent": [ 65 | "curl/7.68.0" 66 | ] 67 | }, 68 | "uri": "/date/", 69 | "method": "POST", 70 | "complete": true 71 | } 72 | ``` 73 | 74 | ## Get complete 75 | 76 | ||| 77 | |------------|---------------------------| 78 | | **Method** | GET | 79 | | **Path** | /{correlationId}/complete | 80 | 81 | Will redirect to the complete attempt or 404 82 | 83 | 84 | ## Force complete 85 | 86 | ||| 87 | |------------|---------------------------| 88 | | **Method** | DELET | 89 | | **Path** | /{correlationId} | 90 | 91 | Forcefully mark request as complete and stop processing (including re-queue) 92 | 93 | 94 | ## Get attempt 95 | 96 | ||| 97 | |------------|--------------------------------------| 98 | | **Method** | GET | 99 | | **Path** | /{correlationId}/attempt/{attemptId} | 100 | 101 | Get result of processing request for the defined attempt 102 | 103 | Returns body, code and headers same as processor returned with additional headers: 104 | 105 | * `X-Status` - status of request processing: `complete` or `processing` 106 | * `X-Processed` - `true` to distinguish result 107 | 108 | 109 | 110 | ## Get request 111 | 112 | ||| 113 | |------------|--------------------------| 114 | | **Method** | GET | 115 | | **Path** | /{correlationId}/request | 116 | 117 | Get request same as it was POSTed for the defined ID 118 | 119 | Returns body and headers same as processor got from client with additional headers: 120 | 121 | * `X-Method` - request method (currently always `POST`) 122 | * `X-Request-Uri` - request URI 123 | * `Last-Modified` - time of creation 124 | -------------------------------------------------------------------------------- /_docs/authorization.md: -------------------------------------------------------------------------------- 1 | # Authorization 2 | 3 | By-default - authorization disabled. Multiple policies allowed. 4 | To allow request at least one policy should be passed. 5 | Each authorization policy can enabled by `enable: yes` param. 6 | 7 | Section in `server.yaml`: `authorization` 8 | 9 | ## JWT 10 | 11 | *section: `authorization.jwt`* 12 | 13 | [Overview](https://jwt.io/) 14 | 15 | HMAC 256 signature validation against secret key 16 | 17 | Configurable parameters: 18 | 19 | * `header` (optional, string, default: `Authorization`) - header that contains JWT 20 | * `secret` (required, string) - secret key to validate signature 21 | 22 | Example minimal unit config 23 | 24 | ```yaml 25 | command: 'echo hello world' 26 | authorization: 27 | jwt: 28 | enable: yes 29 | secret: '$eCrEtKey' 30 | ``` 31 | 32 | ## Query token 33 | 34 | *section: `authorization.query_token`* 35 | 36 | Plain token in a query string. Will be matched against list of allowed tokens. 37 | 38 | For example, client can invoke endpoint by addition token query: `http://example.com/app/?token=deadbeaf` 39 | 40 | Configurable parameters: 41 | 42 | * `param` (optional, string, default: `token`) - query param where token should be placed 43 | * `tokens` (required, []string) - list of allowed tokens 44 | 45 | Example minimal unit config with 3 tokens 46 | 47 | ```yaml 48 | command: 'echo hello world' 49 | authorization: 50 | query_token: 51 | enable: yes 52 | tokens: 53 | - my-token-1 54 | - his-token-2 55 | - deadbeaf 56 | ``` 57 | 58 | ## Header token 59 | 60 | *section: `authorization.header_token`* 61 | 62 | Plain token in a header. Will be matched against list of allowed tokens. 63 | 64 | For example, client can invoke endpoint by curl: 65 | 66 | curl -H 'X-Api-Token: deadbeaf' http://example.com/app/ 67 | 68 | Configurable parameters: 69 | 70 | * `header` (optional, string, default: `X-Api-Token`) - header name where token should be placed 71 | * `tokens` (required, []string) - list of allowed tokens 72 | 73 | Example minimal unit config with 3 tokens 74 | 75 | ```yaml 76 | command: 'echo hello world' 77 | authorization: 78 | header_token: 79 | enable: yes 80 | tokens: 81 | - my-token-1 82 | - his-token-2 83 | - deadbeaf 84 | ``` 85 | 86 | ## Basic 87 | 88 | *section: `authorization.basic`* 89 | 90 | Basic authentication. [Overview](https://en.wikipedia.org/wiki/Basic_access_authentication) 91 | 92 | For example, client can invoke endpoint by curl: 93 | 94 | curl -u 'alice:admin' http://example.com/app/ 95 | 96 | To [calculate](https://unix.stackexchange.com/a/419855) hash you may use `htpasswd` (Debian/Ubuntu: `sudo apt install apache2-utils`) 97 | 98 | htpasswd -bnBC 10 "" password | tr -d ':' 99 | 100 | where `passsword` is a desired password for the user. 101 | 102 | Configurable parameters: 103 | 104 | * `users` (string->string, required) - map of users and their hashed password by bcrypt 105 | 106 | Example minimal config: 107 | 108 | ```yaml 109 | command: 'echo hello world' 110 | authorization: 111 | basic: 112 | enable: yes 113 | users: 114 | alice: '$2y$10$cUe3n8NHaxee.AaGzT8wF.nirPnjv5YLEQGTsLiiMiUAknM2aF2FS' 115 | bob: '$2y$10$iSczi.MlKTrMv3h0Zf.GDeW1NS6ZWxBgtj4ytrKKDrR2s2wIxq5Qa' 116 | ``` 117 | -------------------------------------------------------------------------------- /_docs/cron.md: -------------------------------------------------------------------------------- 1 | # Cron 2 | 3 | 4 | Cron-like jobs allowed as part of Unit definition [thanks to robfig/cron](https://godoc.org/github.com/robfig/cron#hdr-Usage). 5 | 6 | **Example definition:** 7 | 8 | ```yaml 9 | # ... unit definition above ... 10 | cron: 11 | # every hour on the half hour 12 | - spec: 30 * * * * 13 | # same as above but with name to detect in UI and logs 14 | - spec: 30 * * * * 15 | name: named schedule 16 | # each hour with custom payload and headers 17 | - spec: "@hourly" 18 | content: | 19 | hello world 20 | headers: 21 | X-Some-Header: test-header 22 | # each day with content from file 23 | - spec: "@daily" 24 | content_file: /path/to/content 25 | ``` 26 | 27 | 28 | Schema: 29 | 30 | * `spec` (required, string) - cron tab specification for the time interval. See [online builder](https://crontab.guru/) 31 | * `name` (optional, string) - name for entry to distinguish record in UI or in logs. 32 | * `headers` (optional, map string=>string) - headers that will be used in simulated request. 33 | * `content` (optional, string) - simulated request content. 34 | * `content_file` (optional, string) - simulated request content file. Has less priority than `content`. 35 | 36 | Caveats: 37 | 38 | * **Security:** cron job ignores authorizations defined on unit level. 39 | * **Enqueuing:** cron job will be enqueued regardless of status of previous job. 40 | * **Errors:** if cron job can't create request (ex: `content_file` not available) - it will print error to log and 41 | will try again later at the next schedule. -------------------------------------------------------------------------------- /_docs/docker.md: -------------------------------------------------------------------------------- 1 | # Docker 2 | 3 | 4 | Check images in [releases](https://github.com/reddec/nano-run/releases) 5 | 6 | * Latest one: `reddec/nano-run:latest` 7 | 8 | 9 | Create Dockerfile inherited from the image and copy configuration and binaries 10 | 11 | ## Minimal example 12 | 13 | **app.yaml** 14 | ```yaml 15 | command: '/mybinary --with --some args' 16 | ``` 17 | 18 | **Dockerfile** 19 | ```dockerfile 20 | FROM reddec/nano-run 21 | COPY app.yaml /conf.d/app.yaml 22 | COPY mybinary /mybinary 23 | ``` 24 | 25 | **Build & Run** 26 | 27 | ```bash 28 | docker run --rm -p 127.0.0.1:8080:80 $(docker build -q .) 29 | ``` 30 | 31 | Check it's working by sending test request 32 | 33 | ``` 34 | curl -v -X POST "http://127.0.0.1:8080/app/" 35 | ``` 36 | 37 | * To keep tasks persistent - mount `/data` volume like: 38 | `docker run -v $(pwd)/data:data ...` -------------------------------------------------------------------------------- /_docs/flow.md: -------------------------------------------------------------------------------- 1 | # High-level overview 2 | 3 | Subjects: 4 | 5 | * Client - the side which is making HTTP(S) request to the System 6 | * System - instance of nano-run that routing request 7 | * Worker - executable that implements business logic 8 | 9 | During restart - all incomplete tasks will queued again. 10 | 11 | ![image](https://user-images.githubusercontent.com/6597086/92712138-d8b58580-f38b-11ea-8a26-251df5c4ae13.png) 12 | 13 | ![image](https://user-images.githubusercontent.com/6597086/92578247-3085bb00-f2be-11ea-87de-e2c9d94a21fa.png) 14 | 15 | Worker will call upstream with additional headers: 16 | 17 | - `X-Correlation-Id` - request id 18 | - `X-Attempt-Id` - attempt id 19 | - `X-Attempt` - attempt num -------------------------------------------------------------------------------- /_docs/flow.txt: -------------------------------------------------------------------------------- 1 | title Request processing 2 | 3 | Client->System: HTTP(s) request 4 | System->System: Save request 5 | System->Client: Correlation ID 6 | loop attempts 7 | System->Worker: Request 8 | alt request failed 9 | Worker->System: failed 10 | System->System: requeue 11 | else success 12 | Worker->System: success 13 | end 14 | System->System: save attemp 15 | end 16 | 17 | System->System: mark request as complete 18 | Client->System: get result 19 | System->Client: info -------------------------------------------------------------------------------- /_docs/goals.md: -------------------------------------------------------------------------------- 1 | # Goals 2 | 3 | * Minimal requirements for host; 4 | * Should have semi-constant resource consumption regardless of: 5 | * number of requests, 6 | * size of requests, 7 | * kind of requests; 8 | * Should be ready to run without configuration; 9 | * Should be ready for deploying in clouds; 10 | * Should support extending for another providers; 11 | * Can be used as library and as a complete solution; 12 | * Performance has less priority than resource usage. -------------------------------------------------------------------------------- /_docs/internals/worker_flow: -------------------------------------------------------------------------------- 1 | 7Vzdc5s4EP9rPNN76A3f4MdzmjQP6U2veWjvUQbZpgZEhRzb99efBMJ8CNtyEiNIM9OZWgJ9sLu/3dXuKhPzJt59xiBdfUEBjCaGFuwm5qeJYeiWYUzYPy3YFz2uNS06ljgM+EtVx2P4H+SdGu/dhAHMGi8ShCISps1OHyUJ9EmjD2CMts3XFihqrpqCJRQ6Hn0Qib3fw4Csil7PcKv+exguV+XKusO/Lwbly/xLshUI0LbWZd5OzBuMECl+xbsbGDHilXQpxt0deXrYGIYJkRnw/Uv02VqvH2ffluv7qf3wz0OQfHStYponEG34F2P48dcG0halVkQ3XGye7EuKZNswjkBCW7OMYLSGNyhCOH9mBs7csR32hL90F0ZR+TxB+aBFrYsOWSwWhu/T/gBkK8j2qvPGV0AIxEneY2iUXjO+V4gJ3B2lgn6gLRVKiGJI8J6+wgfYFmcHl0ezbG8r7poly1Y1ztqlRAIuUcvD3IflvlEJBMkygtV6ltlcz9HF9aYdyxlmczUQMVoAAmdokwRZndP0R+1Lq66c/xfIgmN2yEIpChkBlOqBIA2YbefAtu0qJPAxBT57uqUagfatSBzxx1L8OyGnR7naInIHT/UunjonWNqg7sWktAVSpigfRkC2ZsjCKKb/ldT9UILujxME1oZDYF09hQ2BUjCgmps3ESYrtEQJiG6r3lmTltU7DwilnII/ISF7bobAhqAmfeEuJD/Y8D9t3vq39uTTjs+cN/a8UeyTbe45HKFfiDbYhxK4pQhdwlMzFlIp8hjDCJDwqbm/1+eYK2Ai2/g+zOhCd6LYr1A832QKRd5uivzBjtdE3uuQeO9qEm8JRPptJd6WlXhXqcR7Jw0qStPBGlTdcwZmUR2BlglSCQi9BocKHN2AoKzA+xqKWPMAI9aohuWt/oDkygLJUwqkqcD8LQhJfp7D+S5+bULMwKSRMIbj8KG6Th79YsrVBLLuYTYWK3MMVNp4QDVVCSrXVMrpZ6nPBp8rtqvitKvLctpUymldwDlMSj+EMogFq8KkOo6OQXse4irqtOdoTqDD1ZRTSfy4R4J6PeGnw/1AeA2Z61GESLUPOX4oZUAU5RHwvO2jxN9gSi1/z12TTAzw1OKpEiHQVvQ0sKEXWF2hWM+Ym47zOkHTFvY6TsJWVwxzal0Le7p4GjhwRHWk8ohRL6eZWk0vULUa08WwzKlQpSLbcBlRdfVUnY7FODxbyZdfeE7Jl1g9q+R1pcEaQzwMUZ0KQczPlzAjFRgygjBLHI4DDqZyOBhiailMnqjBYhsASRAxzT0KWlq6rZqWo4kD9+53yqqkkl9nVZKh9NxmiKa5RI0T0W3P5hQ1zpL9UpxJOY0hQ29iqCsH3msqxRrQ0U2ThFBPoQ9ZCJmyoQ+1Vt0UQx8xwOu6TXdAzICQzLM0J1wbWj6K0wiSoRr7dn1Jx+GsX/tkibZedVT5gljj9W2PbHTYVBrzMA2Bixl4Yv4aC0nEKRkoHNr+mt1ha3r217zhGJuR+mulMJ7319QaG7UZlbfAaUO2lqkQiRdwmg/9isKE1FxVu3Xcc+3mFMUX8FGVvJydyJ629EvxhcJEr1XMabljsbjHXFxDvY8rW2b0Yh83H/oXxmBfeyFlspEdl1W96fl5U+2kRLZe1y27JXfF+q8qhaZYB1vLPBbxZZaAbPvctRr0McWdPUe1s2GJBH93vmvyKBuLVut8W0MrdnsTvoU885XWSZsi82OwmzA10laT/CiWdSlQ4Oep5DHEA92OSrhe44GmGGj9ezR4uzpu5CMWjkrc2GICT7XpGxIXLU+Si7XLiSpMn1gzP6rchqE8t/FeedAubTkr8pbS6nZbzFHUipnU3hI5k3to3ZFSXmdgi7mH91si/ZkOW+mpyRPLNO+P1Ja8qOZy4fkwv7Eu1FzOPduyNVnEXXZRvSOv13lP3dSuBS5PdLCqwiiAMWWwMkXlnbHS7rD0lCeqfJ5dqyg6R8Geh8e0eYTmE+VlZ5cRWVeeiS7vML5dV8iTVeElds+qcE+tCjeO4YLOD9joZIFKUPCuUYGi63jQMyjsd1C0tPB5UBhKQSE6tVUSpbIXdI4B3eY6AwvdGZyxUFpWe8k54fq4MGRxobQ81hOjtj7CxdooKRGBIdngZLCeaRsI6l3TAZUrydbG9nNelsePbBGLd6VyJdqs/q5ckcuv/jqfefs/ -------------------------------------------------------------------------------- /_docs/modes.md: -------------------------------------------------------------------------------- 1 | ## BIN 2 | 3 | Binary modes just executes any script in shell (`/bin/sh` by default). You can override shell per-unit 4 | by `shell: /path/to/shell` configuration param. 5 | 6 | Linux's hosts should not worry about "leaking" processes (when process 7 | creates number of background subprocesses) - it wil be cleaned automatically. 8 | 9 | **For non-Linux users** 10 | 11 | To handle a graceful timeout, child should be able to forward signal: basically, use `exec` before last command. 12 | 13 | Danger (but will work), signals may not be handled by foo 14 | 15 | ```yaml 16 | command: "V=1 RAIL=2 foo bar -c -y -z" 17 | ``` 18 | 19 | Good 20 | 21 | ```yaml 22 | command: "V=1 RAIL=2 exec foo bar -c -y -z" 23 | ``` -------------------------------------------------------------------------------- /_docs/nano-run.svg: -------------------------------------------------------------------------------- 1 | NrgoCLIlibnano-run -------------------------------------------------------------------------------- /_docs/ui.md: -------------------------------------------------------------------------------- 1 | # UI 2 | 3 | UI available by default at port 8989, browser will be automatically redirected to `/ui/`. 4 | 5 | Disable ui by `disable_ui: yes` flag in the config. 6 | 7 | If UI directory (from `ui_directory`, default `ui`), embedded UI will be used. 8 | 9 | UI directory supports static files under `static` directory under `/static` prefix. 10 | 11 | All *.html files scanned under `ui_directory` directory as [Go HTML template](https://golang.org/pkg/html/template/) 12 | with additional functions from [Sprig](http://masterminds.github.io/sprig/). Recursive scanning not supported. 13 | 14 | UI embedded by [go-bindata](https://github.com/go-bindata/go-bindata). -------------------------------------------------------------------------------- /_docs/ui_authorization.md: -------------------------------------------------------------------------------- 1 | # UI Authorization 2 | 3 | By default, there is no authorization (anonymous user will be used). 4 | 5 | If list of `auth.users` is not empty, all authorized users will be allowed. 6 | 7 | ## OAuth2 8 | 9 | **This is mostly recommended way** 10 | 11 | Defined in the section: `auth.oauth2` 12 | 13 | * `title` - text that will be used for login button 14 | * `secret` - OAuth2 client secret 15 | * `key` - OAuth2 client ID 16 | * `callback_url` - redirect URL, must point to your sever plus `/ui/auth/oauth2/callback` 17 | * `auth_url` - authenticate URL, different for each provider 18 | * `token_url` - issue token URL, different for each provider 19 | * `profile_url` (optional) - URL that should return user JSON profile on GET request with authorization by token; if not defined login will an empty string 20 | * `login_field` - (required only if `profile_url` set) filed name (should be string) in profile that identifies user (ex: `login`, `username` or `email`) 21 | * `scopes` (optional) - list of OAuth2 scopes 22 | 23 | 24 | Gitea example: 25 | 26 | ```yaml 27 | auth: 28 | oauth2: 29 | title: Gitea 30 | secret: "oauth secret" 31 | key: "oauth key" 32 | callback_url: "https://YOUR-SERVER/ui/auth/oauth2/callback" 33 | auth_url: "https://gitea-server/login/oauth/authorize" 34 | token_url: "https://gitea-server/login/oauth/access_token" 35 | profile_url: "https://gitea-server/api/v1/user" 36 | login_field: "login" 37 | scopes: 38 | - nano-run 39 | users: 40 | - "reddec" 41 | ``` -------------------------------------------------------------------------------- /_docs/unit.md: -------------------------------------------------------------------------------- 1 | # Unit 2 | 3 | * If work dir not defined - temporary directory will be created and removed after execution for each request automatically. 4 | 5 | 6 | Schema: 7 | 8 | * `command` (required, string) - command to execute (will be executed in a shell) 9 | * `user` (optional, string) - custom user as process owner (only `bin` mode and only for linux), usually requires root privileges 10 | * `interval` (optional, interval) - interval between attempts 11 | * `timeout` (optional, interval) - maximum execution timeout (enabled only for bin mode and only if positive) 12 | * `graceful_timeout` (optional, interval) - maximum execution timeout after which SIGINT will be sent (enabled only for bin mode and only if positive). 13 | Ie: how long to let command react on SIGTERM signal. 14 | * `shell` (optional, string) - shell to execute command in bin mode (default - /bin/sh) 15 | * `environment` (optional, map string=>string) - custom environment for executable (in addition to system) 16 | * `max_request` (optional, integer) - maximum HTTP body size (enabled if positive) 17 | * `attempts` (optional, integer) - maximum number of attempts 18 | * `workers` (optional, integer) - concurrency level - number of parallel requests 19 | * `mode` (optional, string) - execution mode: `bin` or `cgi` 20 | * `workdir` (optional, string) - working directory for the worker. if empty - temporary one will be generated automatically. 21 | * `authorization` (optional, [Authorization](authorization.md)) - request authorization 22 | * `cron` (optional,[Cron](cron.md)) - scheduled requests 23 | * `private` (optional, bool) - do not expose over API, could be used for cron-only jobs -------------------------------------------------------------------------------- /_footer.md: -------------------------------------------------------------------------------- 1 | For Ubuntu/Debian (should be for all LTS) 2 | 3 | ```bash 4 | sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 379CE192D401AB61 5 | echo "deb https://dl.bintray.com/reddec/debian all main" | sudo tee /etc/apt/sources.list.d/reddec.list 6 | sudo apt update 7 | sudo apt install nano-run 8 | ``` 9 | 10 | Ansible snippet 11 | 12 | ```yaml 13 | - name: Add an apt key by id from a keyserver 14 | become: yes 15 | apt_key: 16 | keyserver: keyserver.ubuntu.com 17 | id: 379CE192D401AB61 18 | - name: Add repository 19 | become: yes 20 | apt_repository: 21 | repo: deb https://dl.bintray.com/reddec/debian all main 22 | state: present 23 | filename: reddec 24 | - name: Install nano-run 25 | become: yes 26 | apt: 27 | name: nano-run 28 | update_cache: yes 29 | state: latest 30 | ``` -------------------------------------------------------------------------------- /bundle/debian/nano-run.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Lightweight async request processor 3 | 4 | [Service] 5 | ExecStart=/usr/local/bin/nano-run server run -c /etc/nano-run/server.yaml 6 | Restart=always 7 | RestartSec=3 8 | Environment=GIN_MODE=release 9 | 10 | [Install] 11 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /bundle/debian/postinstall.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SERVICE="nano-run" 4 | RUNNING_USER="${SERVICE}" 5 | 6 | if ! id -u ${RUNNING_USER}; then 7 | echo "Creating user ${RUNNING_USER}..." 8 | useradd -M -c "${RUNNING_USER} dummy user" -r -s /bin/nologin ${RUNNING_USER} 9 | fi 10 | chown -R nano-run:nano-run /var/nano-run/ 11 | systemctl enable "${SERVICE}".service || echo "failed to enable service" 12 | systemctl start "${SERVICE}".service || echo "failed to start service" 13 | -------------------------------------------------------------------------------- /bundle/debian/preremove.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SERVICE="nano-run" 4 | RUNNING_USER="${SERVICE}" 5 | 6 | systemctl stop "${SERVICE}".service || echo "failed to stop service" 7 | systemctl disable "${SERVICE}".service || echo "failed to disable service" 8 | 9 | if id -u ${RUNNING_USER}; then 10 | echo "Removing user ${RUNNING_USER}..." 11 | userdel -r ${RUNNING_USER} 12 | fi 13 | 14 | 15 | -------------------------------------------------------------------------------- /bundle/debian/server.yaml: -------------------------------------------------------------------------------- 1 | # Location to store tasks, blobs and queues 2 | working_directory: /var/nano-run 3 | ui_directory: /var/nano-run/ui 4 | config_directory: /etc/nano-run/conf.d 5 | bind: 127.0.0.1:8989 6 | graceful_shutdown: 5s 7 | max_request: 1048576 8 | -------------------------------------------------------------------------------- /bundle/docker/server.yaml: -------------------------------------------------------------------------------- 1 | working_directory: /data 2 | config_directory: /conf.d 3 | ui_directory: /ui 4 | bind: ":80" 5 | graceful_shutdown: 5s 6 | max_request: 1048576 7 | -------------------------------------------------------------------------------- /cmd/nano-run/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | 8 | "github.com/jessevdk/go-flags" 9 | ) 10 | 11 | var ( 12 | version = "dev" 13 | commit = "dev" 14 | ) 15 | 16 | type Config struct { 17 | Run runCmd `command:"run" description:"run single unit"` 18 | Server serverCmd `command:"server" description:"manage server"` 19 | } 20 | 21 | func main() { 22 | if len(os.Args) == 1 { 23 | os.Args = append(os.Args, "server", "run") 24 | } 25 | var cfg Config 26 | parser := flags.NewParser(&cfg, flags.Default) 27 | parser.LongDescription = "Async webhook processor with minimal system requirements.\n\n" + 28 | "Author: Baryshnikov Aleksandr \n" + 29 | "Source code: https://github.com/reddec/nano-run\n" + 30 | "License: Apache 2.0\n" + 31 | "Version: " + version + "\n" + 32 | "Revision: " + commit 33 | _, err := parser.Parse() 34 | if err != nil { 35 | os.Exit(1) 36 | } 37 | } 38 | 39 | func SignalContext() context.Context { 40 | gctx, closer := context.WithCancel(context.Background()) 41 | go func() { 42 | c := make(chan os.Signal, 1) 43 | signal.Notify(c, signals...) 44 | for range c { 45 | closer() 46 | break 47 | } 48 | }() 49 | return gctx 50 | } 51 | -------------------------------------------------------------------------------- /cmd/nano-run/run_cmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | runtime "runtime" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "nano-run/server" 13 | "nano-run/server/runner" 14 | ) 15 | 16 | type runCmd struct { 17 | Directory string `long:"directory" short:"d" env:"DIRECTORY" description:"Data directory" default:"run"` 18 | UI string `long:"ui" env:"UI" description:"Path to UI directory" default:"templates"` 19 | Interval time.Duration `long:"interval" short:"i" env:"INTERVAL" description:"Requeue interval" default:"3s"` 20 | Attempts int `long:"attempts" short:"a" env:"ATTEMPTS" description:"Max number of attempts" default:"5"` 21 | Concurrency int `long:"concurrency" short:"c" env:"CONCURRENCY" description:"Number of parallel worker (0 - mean number of CPU)" default:"0"` 22 | Mode string `long:"mode" short:"m" env:"MODE" description:"Running mode" default:"bin" choice:"bin" choice:"cgi" choice:"proxy"` 23 | Bind string `long:"bind" short:"b" env:"BIND" description:"Binding address" default:"127.0.0.1:8989"` 24 | Args struct { 25 | Executable string `arg:"executable" description:"path to binary to invoke or url" required:"yes"` 26 | Args []string `arg:"args" description:"executable args"` 27 | } `positional-args:"yes"` 28 | } 29 | 30 | func (cfg *runCmd) Execute([]string) error { 31 | tmpDir, err := ioutil.TempDir("", "") 32 | if err != nil { 33 | return err 34 | } 35 | defer os.RemoveAll(tmpDir) 36 | srv := runner.DefaultConfig() 37 | srv.Bind = cfg.Bind 38 | srv.WorkingDirectory = cfg.Directory 39 | srv.UIDirectory = cfg.UI 40 | srv.ConfigDirectory = tmpDir 41 | 42 | unit := server.DefaultUnit() 43 | 44 | var params []string 45 | params = append(params, strconv.Quote(cfg.Args.Executable)) 46 | for _, arg := range cfg.Args.Args { 47 | params = append(params, strconv.Quote(arg)) 48 | } 49 | unit.Command = strings.Join(params, " ") 50 | unit.WorkDir, _ = os.Getwd() 51 | unit.Attempts = cfg.Attempts 52 | unit.Interval = cfg.Interval 53 | unit.Workers = cfg.concurrency() 54 | unit.Mode = cfg.Mode 55 | 56 | err = unit.SaveFile(filepath.Join(tmpDir, "main.yaml")) 57 | if err != nil { 58 | return err 59 | } 60 | return srv.Run(SignalContext()) 61 | } 62 | 63 | func (cfg runCmd) concurrency() int { 64 | if cfg.Concurrency <= 0 { 65 | return runtime.NumCPU() 66 | } 67 | return cfg.Concurrency 68 | } 69 | -------------------------------------------------------------------------------- /cmd/nano-run/server_cmd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path/filepath" 7 | 8 | "nano-run/server" 9 | "nano-run/server/runner" 10 | ) 11 | 12 | type serverCmd struct { 13 | Run serverRunCmd `command:"run" description:"run server"` 14 | Init serverInitCmd `command:"init" description:"initialize server"` 15 | } 16 | 17 | type serverInitCmd struct { 18 | Directory string `short:"d" long:"directory" env:"DIRECTORY" description:"Target directory" default:"."` 19 | ConfigFile string `long:"config-file" env:"CONFIG_FILE" description:"Config file name" default:"server.yaml"` 20 | NoSample bool `long:"no-sample" env:"NO_SAMPLE" description:"Do not create same file"` 21 | } 22 | 23 | func (cmd *serverInitCmd) Execute([]string) error { 24 | err := os.MkdirAll(cmd.Directory, 0755) 25 | if err != nil { 26 | return err 27 | } 28 | cfg := runner.DefaultConfig() 29 | err = cfg.SaveFile(filepath.Join(cmd.Directory, cmd.ConfigFile)) 30 | if err != nil { 31 | return err 32 | } 33 | err = os.MkdirAll(filepath.Join(cmd.Directory, cfg.ConfigDirectory), 0755) 34 | if err != nil { 35 | return err 36 | } 37 | err = os.MkdirAll(filepath.Join(cmd.Directory, cfg.WorkingDirectory), 0755) 38 | if err != nil { 39 | return err 40 | } 41 | if !cmd.NoSample { 42 | unit := server.DefaultUnit() 43 | err = unit.SaveFile(filepath.Join(cmd.Directory, cfg.ConfigDirectory, "sample.yaml")) 44 | if err != nil { 45 | return err 46 | } 47 | } 48 | return nil 49 | } 50 | 51 | type serverRunCmd struct { 52 | Fail bool `short:"f" long:"fail" env:"FAIL" description:"Fail if no config file"` 53 | Config string `short:"c" long:"config" env:"CONFIG" description:"Configuration file" default:"server.yaml"` 54 | } 55 | 56 | func (cmd *serverRunCmd) Execute([]string) error { 57 | cfg := runner.DefaultConfig() 58 | err := cfg.LoadFile(cmd.Config) 59 | if os.IsNotExist(err) && !cmd.Fail { 60 | log.Println("no config file found - using transient default configuration") 61 | cfg.ConfigDirectory = filepath.Join("run", "conf.d") 62 | cfg.WorkingDirectory = filepath.Join("run", "data") 63 | err := cfg.CreateDirs() 64 | if err != nil { 65 | return err 66 | } 67 | } else if err != nil { 68 | return err 69 | } 70 | log.Println("configuration loaded") 71 | return cfg.Run(SignalContext()) 72 | } 73 | -------------------------------------------------------------------------------- /cmd/nano-run/signals_default.go: -------------------------------------------------------------------------------- 1 | // +build !darwin,!linux 2 | 3 | package main 4 | 5 | import ( 6 | "os" 7 | ) 8 | 9 | var signals = []os.Signal{os.Interrupt} 10 | -------------------------------------------------------------------------------- /cmd/nano-run/signals_posix.go: -------------------------------------------------------------------------------- 1 | // +build linux darwin 2 | 3 | package main 4 | 5 | import ( 6 | "os" 7 | "syscall" 8 | ) 9 | 10 | var signals = []os.Signal{syscall.SIGTERM, os.Interrupt} 11 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module nano-run 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/Masterminds/goutils v1.1.0 // indirect 7 | github.com/Masterminds/semver v1.5.0 // indirect 8 | github.com/Masterminds/sprig v2.22.0+incompatible 9 | github.com/dgraph-io/badger v1.6.1 10 | github.com/dgrijalva/jwt-go v3.2.0+incompatible 11 | github.com/gin-gonic/gin v1.6.3 12 | github.com/google/uuid v1.1.2 13 | github.com/huandu/xstrings v1.3.2 // indirect 14 | github.com/imdario/mergo v0.3.11 // indirect 15 | github.com/jessevdk/go-flags v1.4.1-0.20200711081900-c17162fe8fd7 16 | github.com/mitchellh/copystructure v1.0.0 // indirect 17 | github.com/robfig/cron/v3 v3.0.0 18 | github.com/stretchr/testify v1.4.0 19 | golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a 20 | golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 21 | gopkg.in/yaml.v2 v2.3.0 22 | ) 23 | -------------------------------------------------------------------------------- /internal/nano_logger.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/dgraph-io/badger" 7 | ) 8 | 9 | func NanoLogger(wrap *log.Logger) badger.Logger { 10 | return &nanoLogger{logger: wrap} 11 | } 12 | 13 | type nanoLogger struct { 14 | logger *log.Logger 15 | } 16 | 17 | func (nl *nanoLogger) Errorf(s string, i ...interface{}) { 18 | nl.logger.Printf("[error] "+s, i...) 19 | } 20 | 21 | func (nl *nanoLogger) Warningf(s string, i ...interface{}) { 22 | nl.logger.Printf("[warn] "+s, i...) 23 | } 24 | 25 | func (nl *nanoLogger) Infof(s string, i ...interface{}) { 26 | nl.logger.Printf("[info] "+s, i...) 27 | } 28 | 29 | func (nl *nanoLogger) Debugf(s string, i ...interface{}) { 30 | nl.logger.Printf("[debug] "+s, i...) 31 | } 32 | -------------------------------------------------------------------------------- /server/api/adapter.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "net/http" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/gin-gonic/gin" 11 | 12 | "nano-run/services/meta" 13 | "nano-run/worker" 14 | ) 15 | 16 | // Expose worker as HTTP handler: 17 | // POST / - post task async, returns 303 See Other and location 18 | // PUT / - post task synchronously, supports ?wait= parameter for custom wait time 19 | // GET /:id - get task info. 20 | // POST /:id - retry task, redirects to /:id 21 | // GET /:id/completed - redirect to completed attempt (or 404 if task not yet complete) 22 | // GET /:id/attempt/:atid - get attempt result (as-is). 23 | // GET /:id/request - replay request (as-is). 24 | func Expose(router gin.IRouter, wrk *worker.Worker, defaultWaitTime time.Duration) { 25 | handler := &workerHandler{wrk: wrk, defaultWait: defaultWaitTime} 26 | router.POST("/", func(gctx *gin.Context) { 27 | id, err := wrk.Enqueue(gctx.Request) 28 | if err != nil { 29 | log.Println("failed to enqueue:", err) 30 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 31 | return 32 | } 33 | gctx.Header("X-Correlation-Id", id) 34 | gctx.Redirect(http.StatusSeeOther, id+"?"+gctx.Request.URL.RawQuery) 35 | }) 36 | router.PUT("/", handler.createSyncTask) 37 | taskRoutes := router.Group("/:id") 38 | taskRoutes.GET("", handler.getTask) 39 | taskRoutes.POST("", handler.retry) 40 | taskRoutes.DELETE("", handler.completeRequest) 41 | taskRoutes.GET("/completed", handler.getComplete) 42 | // get attempt result as-is. 43 | taskRoutes.GET("/attempt/:attemptId", handler.getAttempt) 44 | // get recorded request. 45 | taskRoutes.GET("/request", handler.getRequest) 46 | } 47 | 48 | type workerHandler struct { 49 | wrk *worker.Worker 50 | defaultWait time.Duration 51 | } 52 | 53 | func (wh *workerHandler) createSyncTask(gctx *gin.Context) { 54 | var queryParams struct { 55 | Wait time.Duration `query:"wait" form:"wait"` 56 | } 57 | queryParams.Wait = wh.defaultWait 58 | 59 | if err := gctx.BindQuery(&queryParams); err != nil { 60 | return 61 | } 62 | if queryParams.Wait <= 0 { 63 | gctx.AbortWithStatus(http.StatusBadRequest) 64 | return 65 | } 66 | 67 | tracker, err := wh.wrk.EnqueueWithTracker(gctx.Request) 68 | if err != nil { 69 | log.Println("failed to enqueue:", err) 70 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 71 | return 72 | } 73 | gctx.Header("X-Correlation-Id", tracker.ID()) 74 | select { 75 | case <-tracker.Done(): 76 | gctx.Redirect(http.StatusSeeOther, tracker.ID()+"/completed?"+gctx.Request.URL.RawQuery) 77 | case <-time.After(queryParams.Wait): 78 | gctx.AbortWithStatus(http.StatusGatewayTimeout) 79 | } 80 | } 81 | 82 | // get request meta information. 83 | func (wh *workerHandler) getTask(gctx *gin.Context) { 84 | requestID := gctx.Param("id") 85 | info, err := wh.wrk.Meta().Get(requestID) 86 | if err != nil { 87 | log.Println("failed access request", requestID, ":", err) 88 | gctx.AbortWithStatus(http.StatusNotFound) 89 | return 90 | } 91 | gctx.Header("X-Correlation-Id", requestID) 92 | gctx.Header("Content-Version", strconv.Itoa(len(info.Attempts))) 93 | // modification time 94 | setLastModify(gctx, info) 95 | 96 | gctx.Header("Age", strconv.FormatInt(int64(time.Since(info.CreatedAt)/time.Second), 10)) 97 | if info.Complete { 98 | gctx.Header("X-Status", "complete") 99 | } else { 100 | gctx.Header("X-Status", "processing") 101 | } 102 | 103 | if len(info.Attempts) > 0 { 104 | gctx.Header("X-Last-Attempt", info.Attempts[len(info.Attempts)-1].ID) 105 | gctx.Header("X-Last-Attempt-At", info.Attempts[len(info.Attempts)-1].CreatedAt.Format(time.RFC850)) 106 | } 107 | 108 | if info.Complete { 109 | lastAttempt := info.Attempts[len(info.Attempts)-1] 110 | gctx.Request.URL.Path += "/attempt/" + lastAttempt.ID 111 | gctx.Header("Location", gctx.Request.URL.String()) 112 | } 113 | gctx.IndentedJSON(http.StatusOK, info) 114 | } 115 | 116 | func (wh *workerHandler) retry(gctx *gin.Context) { 117 | requestID := gctx.Param("id") 118 | id, err := wh.wrk.Retry(gctx.Request.Context(), requestID) 119 | if err != nil { 120 | log.Println("failed to retry:", err) 121 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 122 | return 123 | } 124 | gctx.Header("X-Correlation-Id", id) 125 | gctx.Redirect(http.StatusSeeOther, id+"?"+gctx.Request.URL.RawQuery) 126 | } 127 | 128 | func (wh *workerHandler) completeRequest(gctx *gin.Context) { 129 | requestID := gctx.Param("id") 130 | info, err := wh.wrk.Meta().Get(requestID) 131 | if err != nil { 132 | log.Println("failed access request", requestID, ":", err) 133 | gctx.AbortWithStatus(http.StatusNotFound) 134 | return 135 | } 136 | if !info.Complete { 137 | err = wh.wrk.Meta().Complete(requestID) 138 | if err != nil { 139 | log.Println("failed to mark request as complete:", err) 140 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 141 | return 142 | } 143 | } 144 | gctx.AbortWithStatus(http.StatusNoContent) 145 | } 146 | 147 | func (wh *workerHandler) getComplete(gctx *gin.Context) { 148 | requestID := gctx.Param("id") 149 | info, err := wh.wrk.Meta().Get(requestID) 150 | if err != nil { 151 | log.Println("failed access request", requestID, ":", err) 152 | gctx.AbortWithStatus(http.StatusNotFound) 153 | return 154 | } 155 | if !info.Complete { 156 | gctx.AbortWithStatus(http.StatusTooEarly) 157 | return 158 | } 159 | lastAttempt := info.Attempts[len(info.Attempts)-1] 160 | gctx.Redirect(http.StatusMovedPermanently, "attempt/"+lastAttempt.ID) 161 | } 162 | 163 | func (wh *workerHandler) getAttempt(gctx *gin.Context) { 164 | requestID := gctx.Param("id") 165 | attemptID := gctx.Param("attemptId") 166 | info, err := wh.wrk.Meta().Get(requestID) 167 | if err != nil { 168 | log.Println("failed access request", requestID, ":", err) 169 | gctx.AbortWithStatus(http.StatusNotFound) 170 | return 171 | } 172 | var attempt meta.Attempt 173 | var found bool 174 | for _, atp := range info.Attempts { 175 | if atp.ID == attemptID { 176 | found = true 177 | attempt = atp 178 | break 179 | } 180 | } 181 | if !found { 182 | gctx.AbortWithStatus(http.StatusNotFound) 183 | return 184 | } 185 | body, err := wh.wrk.Blobs().Get(attempt.ID) 186 | if err != nil { 187 | log.Println("failed to get body:", err) 188 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 189 | return 190 | } 191 | defer body.Close() 192 | gctx.Header("Last-Modified", attempt.CreatedAt.Format(time.RFC850)) 193 | if info.Complete { 194 | gctx.Header("X-Status", "complete") 195 | } else { 196 | gctx.Header("X-Status", "processing") 197 | } 198 | gctx.Header("X-Processed", "true") 199 | for k, v := range attempt.Headers { 200 | gctx.Request.Header[k] = v 201 | } 202 | _, _ = io.Copy(gctx.Writer, body) 203 | } 204 | 205 | func (wh *workerHandler) getRequest(gctx *gin.Context) { 206 | requestID := gctx.Param("id") 207 | info, err := wh.wrk.Meta().Get(requestID) 208 | if err != nil { 209 | log.Println("failed access request", requestID, ":", err) 210 | gctx.AbortWithStatus(http.StatusNotFound) 211 | return 212 | } 213 | gctx.Header("Last-Modified", info.CreatedAt.Format(time.RFC850)) 214 | f, err := wh.wrk.Blobs().Get(requestID) 215 | if err != nil { 216 | log.Println("failed to get data:", err) 217 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 218 | return 219 | } 220 | defer f.Close() 221 | 222 | gctx.Header("X-Method", info.Method) 223 | gctx.Header("X-Request-Uri", info.URI) 224 | 225 | for k, v := range info.Headers { 226 | gctx.Request.Header[k] = v 227 | } 228 | gctx.Status(http.StatusOK) 229 | _, _ = io.Copy(gctx.Writer, f) 230 | } 231 | 232 | func setLastModify(gctx *gin.Context, info *meta.Request) { 233 | if info.Complete { 234 | gctx.Header("Last-Modified", info.CompleteAt.Format(time.RFC850)) 235 | } else if len(info.Attempts) > 0 { 236 | gctx.Header("Last-Modified", info.Attempts[len(info.Attempts)-1].CreatedAt.Format(time.RFC850)) 237 | } else { 238 | gctx.Header("Last-Modified", info.CreatedAt.Format(time.RFC850)) 239 | } 240 | } 241 | -------------------------------------------------------------------------------- /server/auth.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | "sort" 7 | 8 | "github.com/dgrijalva/jwt-go" 9 | "github.com/gin-gonic/gin" 10 | "golang.org/x/crypto/bcrypt" 11 | ) 12 | 13 | func (cfg Unit) enableAuthorization() func(gctx *gin.Context) { 14 | var handlers []AuthHandlerFunc 15 | if cfg.Authorization.JWT.Enable { 16 | handlers = append(handlers, cfg.Authorization.JWT.Create()) 17 | } 18 | if cfg.Authorization.QueryToken.Enable { 19 | handlers = append(handlers, cfg.Authorization.QueryToken.Create()) 20 | } 21 | if cfg.Authorization.HeaderToken.Enable { 22 | handlers = append(handlers, cfg.Authorization.HeaderToken.Create()) 23 | } 24 | if cfg.Authorization.Basic.Enable { 25 | handlers = append(handlers, cfg.Authorization.Basic.Create()) 26 | } 27 | return func(gctx *gin.Context) { 28 | var authorized = len(handlers) == 0 29 | for _, h := range handlers { 30 | if h(gctx.Request) { 31 | authorized = true 32 | break 33 | } 34 | } 35 | if !authorized { 36 | gctx.AbortWithStatus(http.StatusForbidden) 37 | return 38 | } 39 | gctx.Next() 40 | } 41 | } 42 | 43 | type AuthHandlerFunc func(req *http.Request) bool 44 | 45 | type JWT struct { 46 | Header string `yaml:"header"` // JWT header - by default Authorization 47 | Secret string `yaml:"secret"` // key to verify JWT 48 | } 49 | 50 | func (cfg JWT) GetHeader() string { 51 | if cfg.Header == "" { 52 | return "Authorization" 53 | } 54 | return cfg.Header 55 | } 56 | 57 | func (cfg JWT) Create() AuthHandlerFunc { 58 | header := cfg.GetHeader() 59 | 60 | return func(req *http.Request) bool { 61 | rawToken := req.Header.Get(header) 62 | t, err := jwt.Parse(rawToken, func(token *jwt.Token) (interface{}, error) { 63 | if token.Method != jwt.SigningMethodHS256 { 64 | return nil, errors.New("unknown method") 65 | } 66 | return []byte(cfg.Secret), nil 67 | }) 68 | return err == nil && t.Valid 69 | } 70 | } 71 | 72 | type QueryToken struct { 73 | Param string `yaml:"param"` // query name - by default 'token' 74 | Tokens []string `yaml:"tokens"` // allowed tokens 75 | } 76 | 77 | func (cfg QueryToken) GetParam() string { 78 | if cfg.Param == "" { 79 | return "token" 80 | } 81 | return cfg.Param 82 | } 83 | 84 | func (cfg QueryToken) Create() AuthHandlerFunc { 85 | param := cfg.GetParam() 86 | tokens := map[string]bool{} 87 | for _, k := range cfg.Tokens { 88 | tokens[k] = true 89 | } 90 | return func(req *http.Request) bool { 91 | token := req.URL.Query().Get(param) 92 | return tokens[token] 93 | } 94 | } 95 | 96 | type HeaderToken struct { 97 | Header string `yaml:"header"` // header name - by default X-Api-Token 98 | Tokens []string `yaml:"tokens"` // allowed tokens 99 | } 100 | 101 | func (cfg HeaderToken) GetHeader() string { 102 | if cfg.Header == "" { 103 | return "X-Api-Token" 104 | } 105 | return cfg.Header 106 | } 107 | 108 | func (cfg HeaderToken) Create() AuthHandlerFunc { 109 | header := cfg.GetHeader() 110 | tokens := map[string]bool{} 111 | for _, k := range cfg.Tokens { 112 | tokens[k] = true 113 | } 114 | return func(req *http.Request) bool { 115 | token := req.URL.Query().Get(header) 116 | return tokens[token] 117 | } 118 | } 119 | 120 | type Basic struct { 121 | Users map[string]string `yaml:"users"` // users -> bcrypted password map 122 | } 123 | 124 | func (cfg Basic) Create() AuthHandlerFunc { 125 | return func(req *http.Request) bool { 126 | u, p, ok := req.BasicAuth() 127 | if !ok { 128 | return false 129 | } 130 | h, ok := cfg.Users[u] 131 | if !ok { 132 | return false 133 | } 134 | return bcrypt.CompareHashAndPassword([]byte(h), []byte(p)) == nil 135 | } 136 | } 137 | 138 | func (cfg Basic) Logins() []string { 139 | var ans []string 140 | for name := range cfg.Users { 141 | ans = append(ans, name) 142 | } 143 | sort.Strings(ans) 144 | return ans 145 | } 146 | -------------------------------------------------------------------------------- /server/cron.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "os" 12 | "strconv" 13 | 14 | "github.com/robfig/cron/v3" 15 | 16 | "nano-run/worker" 17 | ) 18 | 19 | type CronSpec struct { 20 | Spec string `yaml:"spec"` // cron tab spec with seconds precision 21 | Name string `yaml:"name,omitempty"` // optional name to distinguish in logs and ui 22 | Headers map[string]string `yaml:"headers,omitempty"` // headers in simulated request 23 | Content string `yaml:"content,omitempty"` // content in simulated request 24 | ContentFile string `yaml:"content_file,omitempty"` // content file to read for request content 25 | } 26 | 27 | func (cs CronSpec) Validate() error { 28 | _, err := cron.ParseStandard(cs.Spec) 29 | return err 30 | } 31 | 32 | func (cs *CronSpec) Label(def string) string { 33 | if cs.Name != "" { 34 | return cs.Name 35 | } 36 | return def 37 | } 38 | 39 | func (cs *CronSpec) Request(ctx context.Context, requestPath string) (*http.Request, error) { 40 | var src io.ReadCloser 41 | if cs.Content != "" || cs.ContentFile == "" { 42 | src = ioutil.NopCloser(bytes.NewReader([]byte(cs.Content))) 43 | } else if f, err := os.Open(cs.ContentFile); err != nil { 44 | return nil, err 45 | } else { 46 | src = f 47 | } 48 | 49 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, requestPath, src) 50 | if err != nil { 51 | _ = src.Close() 52 | } 53 | return req, err 54 | } 55 | 56 | type CronEntry struct { 57 | Name string 58 | Spec CronSpec 59 | Worker *worker.Worker 60 | Config Unit 61 | ID cron.EntryID 62 | ctx context.Context 63 | } 64 | 65 | func (ce *CronEntry) Unit() Unit { return ce.Config } 66 | 67 | // Cron initializes cron engine and registers all required worker schedules to it. 68 | func Cron(ctx context.Context, workers []*worker.Worker, configs []Unit) ([]*CronEntry, *cron.Cron, error) { 69 | engine := cron.New() 70 | var entries []*CronEntry 71 | for i, wrk := range workers { 72 | cfg := configs[i] 73 | for i, cronSpec := range cfg.Cron { 74 | name := cfg.Name() + "/" + cronSpec.Label(strconv.Itoa(i)) 75 | entry := &CronEntry{ 76 | Spec: cronSpec, 77 | Worker: wrk, 78 | Config: cfg, 79 | Name: name, 80 | ctx: ctx, 81 | } 82 | id, err := engine.AddJob(cronSpec.Spec, entry) 83 | if err != nil { 84 | return nil, nil, fmt.Errorf("cron record %s: %w", name, err) 85 | } 86 | entry.ID = id 87 | entries = append(entries, entry) 88 | } 89 | } 90 | return entries, engine, nil 91 | } 92 | 93 | func (ce *CronEntry) Run() { 94 | req, err := ce.Spec.Request(ce.ctx, ce.Config.Path()) 95 | if err != nil { 96 | log.Println("failed create cron", ce.Name, "request:", err) 97 | return 98 | } 99 | id, err := ce.Worker.Enqueue(req) 100 | if err != nil { 101 | log.Println("failed enqueue cron", ce.Name, "job:", err) 102 | return 103 | } 104 | log.Println("enqueued cron", ce.Name, "job with id", id) 105 | } 106 | -------------------------------------------------------------------------------- /server/internal/flags.go: -------------------------------------------------------------------------------- 1 | // +build !linux 2 | 3 | package internal 4 | 5 | import ( 6 | "os" 7 | "os/exec" 8 | ) 9 | 10 | func SetBinFlags(cmd *exec.Cmd) { 11 | 12 | } 13 | 14 | func IntSignal(cmd *exec.Cmd) error { 15 | return cmd.Process.Signal(os.Interrupt) 16 | } 17 | 18 | func KillSignal(cmd *exec.Cmd) error { 19 | return cmd.Process.Signal(os.Kill) 20 | } 21 | -------------------------------------------------------------------------------- /server/internal/flags_linux.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "os/exec" 5 | "syscall" 6 | ) 7 | 8 | func SetBinFlags(cmd *exec.Cmd) { 9 | if cmd.SysProcAttr == nil { 10 | cmd.SysProcAttr = &syscall.SysProcAttr{} 11 | } 12 | cmd.SysProcAttr.Pdeathsig = syscall.SIGTERM 13 | cmd.SysProcAttr.Setpgid = true 14 | } 15 | 16 | func IntSignal(cmd *exec.Cmd) error { 17 | return syscall.Kill(-cmd.Process.Pid, syscall.SIGINT) 18 | } 19 | 20 | func KillSignal(cmd *exec.Cmd) error { 21 | return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) 22 | } 23 | -------------------------------------------------------------------------------- /server/mode_bin.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "io/ioutil" 6 | "log" 7 | "net/http" 8 | "os" 9 | "os/exec" 10 | "strconv" 11 | "strings" 12 | "time" 13 | 14 | "nano-run/server/internal" 15 | ) 16 | 17 | type markerResponse struct { 18 | dataSent bool 19 | res http.ResponseWriter 20 | } 21 | 22 | func (m *markerResponse) Header() http.Header { 23 | return m.res.Header() 24 | } 25 | 26 | func (m *markerResponse) Write(bytes []byte) (int, error) { 27 | m.dataSent = true 28 | return m.res.Write(bytes) 29 | } 30 | 31 | func (m *markerResponse) WriteHeader(statusCode int) { 32 | m.dataSent = true 33 | m.res.WriteHeader(statusCode) 34 | } 35 | 36 | type binHandler struct { 37 | user string 38 | command string 39 | workDir string 40 | shell string 41 | environment []string 42 | timeout time.Duration 43 | gracefulTimeout time.Duration 44 | } 45 | 46 | func (bh *binHandler) ServeHTTP(writer http.ResponseWriter, request *http.Request) { 47 | marker := &markerResponse{res: writer} 48 | ctx := request.Context() 49 | cmd := exec.Command(bh.shell, "-c", bh.command) //nolint:gosec 50 | 51 | if bh.workDir == "" { 52 | tmpDir, err := ioutil.TempDir("", "") 53 | if err != nil { 54 | http.Error(writer, err.Error(), http.StatusInternalServerError) 55 | return 56 | } 57 | defer os.RemoveAll(tmpDir) 58 | cmd.Dir = tmpDir 59 | } else { 60 | cmd.Dir = bh.workDir 61 | } 62 | 63 | var env = bh.cloneEnv() 64 | for k, v := range request.Header { 65 | ke := strings.ToUpper(strings.Replace(k, "-", "_", -1)) 66 | env = append(env, ke+"="+strings.Join(v, ",")) 67 | } 68 | 69 | cmd.Stderr = os.Stderr 70 | cmd.Stdin = request.Body 71 | cmd.Stdout = marker 72 | cmd.Env = env 73 | internal.SetBinFlags(cmd) 74 | err := setUser(cmd, bh.user) 75 | if err != nil { 76 | writer.Header().Set("X-Return-Code", strconv.Itoa(cmd.ProcessState.ExitCode())) 77 | writer.WriteHeader(http.StatusInternalServerError) 78 | return 79 | } 80 | err = bh.run(ctx, cmd) 81 | 82 | if codeReset, ok := writer.(interface{ Status(status int) }); ok && err != nil { 83 | codeReset.Status(http.StatusBadGateway) 84 | } 85 | 86 | if err != nil { 87 | writer.Header().Set("X-Return-Code", strconv.Itoa(cmd.ProcessState.ExitCode())) 88 | writer.WriteHeader(http.StatusBadGateway) 89 | } else { 90 | writer.Header().Set("X-Return-Code", strconv.Itoa(cmd.ProcessState.ExitCode())) 91 | writer.WriteHeader(http.StatusNoContent) 92 | } 93 | } 94 | 95 | func (bh *binHandler) run(global context.Context, cmd *exec.Cmd) error { 96 | err := cmd.Start() 97 | if err != nil { 98 | return err 99 | } 100 | 101 | var ( 102 | gracefulTimeout <-chan time.Time 103 | ctx context.Context 104 | ) 105 | 106 | var gracefulTimer *time.Ticker 107 | if bh.gracefulTimeout > 0 { 108 | gracefulTimer = time.NewTicker(bh.gracefulTimeout) 109 | defer gracefulTimer.Stop() 110 | gracefulTimeout = gracefulTimer.C 111 | } 112 | 113 | if bh.timeout > 0 { 114 | t, cancel := context.WithTimeout(global, bh.timeout) 115 | defer cancel() 116 | ctx = t 117 | } else { 118 | ctx = global 119 | } 120 | 121 | var process = make(chan error, 1) 122 | 123 | go func() { 124 | defer close(process) 125 | process <- cmd.Wait() 126 | }() 127 | 128 | for { 129 | select { 130 | case <-gracefulTimeout: 131 | if err := internal.IntSignal(cmd); err != nil { 132 | log.Println("failed send signal to process:", err) 133 | } else { 134 | log.Println("sent graceful shutdown to process") 135 | } 136 | gracefulTimer.Stop() 137 | gracefulTimeout = nil 138 | case <-ctx.Done(): 139 | if err := internal.KillSignal(cmd); err != nil { 140 | log.Println("failed send kill to process:", err) 141 | } else { 142 | log.Println("sent kill to process") 143 | } 144 | return <-process 145 | case err := <-process: 146 | return err 147 | } 148 | } 149 | } 150 | 151 | func (bh *binHandler) cloneEnv() []string { 152 | var cp = make([]string, len(bh.environment)) 153 | copy(cp, bh.environment) 154 | return cp 155 | } 156 | -------------------------------------------------------------------------------- /server/mode_bin_default.go: -------------------------------------------------------------------------------- 1 | //+build !linux 2 | 3 | package server 4 | 5 | import "os/exec" 6 | 7 | func setUser(cmd *exec.Cmd, user string) error { 8 | return nil 9 | } 10 | -------------------------------------------------------------------------------- /server/mode_bin_linux.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "os/exec" 5 | "os/user" 6 | "strconv" 7 | "syscall" 8 | ) 9 | 10 | func setUser(cmd *exec.Cmd, userName string) error { 11 | if userName == "" { 12 | return nil 13 | } 14 | info, err := user.Lookup(userName) 15 | if err != nil { 16 | return err 17 | } 18 | uid, err := strconv.Atoi(info.Uid) 19 | if err != nil { 20 | return err 21 | } 22 | gid, err := strconv.Atoi(info.Gid) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | if cmd.SysProcAttr == nil { 28 | cmd.SysProcAttr = &syscall.SysProcAttr{} 29 | } 30 | cmd.SysProcAttr.Credential = &syscall.Credential{ 31 | Uid: uint32(uid), 32 | Gid: uint32(gid), 33 | } 34 | return nil 35 | } 36 | -------------------------------------------------------------------------------- /server/runner/handler.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "context" 5 | "html/template" 6 | "io/ioutil" 7 | "log" 8 | "net/http" 9 | "os" 10 | "path/filepath" 11 | "time" 12 | 13 | "github.com/Masterminds/sprig" 14 | "github.com/gin-gonic/gin" 15 | "github.com/robfig/cron/v3" 16 | "gopkg.in/yaml.v2" 17 | 18 | "nano-run/server" 19 | embedded_static "nano-run/server/runner/embedded/static" 20 | embedded_templates "nano-run/server/runner/embedded/templates" 21 | "nano-run/server/ui" 22 | "nano-run/worker" 23 | ) 24 | 25 | //go:generate go-bindata -pkg templates -o embedded/templates/bindata.go -nometadata -prefix ../../templates/ ../../templates/ 26 | //go:generate go-bindata -fs -pkg static -o embedded/static/bindata.go -prefix ../../templates/static/ ../../templates/static/... 27 | 28 | type Config struct { 29 | UIDirectory string `yaml:"ui_directory"` 30 | WorkingDirectory string `yaml:"working_directory"` 31 | ConfigDirectory string `yaml:"config_directory"` 32 | Bind string `yaml:"bind"` 33 | GracefulShutdown time.Duration `yaml:"graceful_shutdown"` 34 | DisableUI bool `yaml:"disable_ui"` 35 | Auth ui.Authorization `yaml:"auth,omitempty"` 36 | DefaultWaitTime time.Duration `yaml:"wait_time,omitempty"` 37 | TLS struct { 38 | Enable bool `yaml:"enable"` 39 | Cert string `yaml:"cert"` 40 | Key string `yaml:"key"` 41 | } `yaml:"tls,omitempty"` 42 | } 43 | 44 | const ( 45 | defaultWaitTime = 30 * time.Second 46 | defaultGracefulShutdown = 5 * time.Second 47 | defaultBind = "127.0.0.1:8989" 48 | ) 49 | 50 | func DefaultConfig() Config { 51 | var cfg Config 52 | cfg.Bind = defaultBind 53 | cfg.WorkingDirectory = filepath.Join("run") 54 | cfg.ConfigDirectory = filepath.Join("conf.d") 55 | cfg.UIDirectory = filepath.Join("ui") 56 | cfg.GracefulShutdown = defaultGracefulShutdown 57 | cfg.DefaultWaitTime = defaultWaitTime 58 | return cfg 59 | } 60 | 61 | func (cfg Config) CreateDirs() error { 62 | err := os.MkdirAll(cfg.WorkingDirectory, 0755) 63 | if err != nil { 64 | return err 65 | } 66 | return os.MkdirAll(cfg.ConfigDirectory, 0755) 67 | } 68 | 69 | func (cfg *Config) LoadFile(file string) error { 70 | data, err := ioutil.ReadFile(file) 71 | if err != nil { 72 | return err 73 | } 74 | err = yaml.Unmarshal(data, cfg) 75 | if err != nil { 76 | return err 77 | } 78 | if !filepath.IsAbs(cfg.WorkingDirectory) { 79 | cfg.WorkingDirectory = filepath.Join(filepath.Dir(file), cfg.WorkingDirectory) 80 | } 81 | if !filepath.IsAbs(cfg.ConfigDirectory) { 82 | cfg.ConfigDirectory = filepath.Join(filepath.Dir(file), cfg.ConfigDirectory) 83 | } 84 | return nil 85 | } 86 | 87 | func (cfg Config) SaveFile(file string) error { 88 | data, err := yaml.Marshal(cfg) 89 | if err != nil { 90 | return err 91 | } 92 | return ioutil.WriteFile(file, data, 0600) 93 | } 94 | 95 | func (cfg Config) Create(global context.Context, defaultWaitTime time.Duration) (*Server, error) { 96 | units, err := server.Units(cfg.ConfigDirectory) 97 | if err != nil { 98 | return nil, err 99 | } 100 | workers, err := server.Workers(cfg.WorkingDirectory, units) 101 | if err != nil { 102 | return nil, err 103 | } 104 | 105 | ctx, cancel := context.WithCancel(global) 106 | 107 | cronEntries, cronEngine, err := server.Cron(ctx, workers, units) 108 | if err != nil { 109 | cancel() 110 | return nil, err 111 | } 112 | 113 | router := gin.Default() 114 | router.Use(func(gctx *gin.Context) { 115 | gctx.Request = gctx.Request.WithContext(global) 116 | gctx.Next() 117 | }) 118 | cfg.installUI(router, units, workers, cronEntries) 119 | server.Attach(router.Group("/api/"), units, workers, defaultWaitTime) 120 | 121 | srv := &Server{ 122 | Handler: router, 123 | workers: workers, 124 | cronEngine: cronEngine, 125 | cronEntries: cronEntries, 126 | units: units, 127 | done: make(chan struct{}), 128 | cancel: cancel, 129 | } 130 | go srv.run(ctx) 131 | return srv, nil 132 | } 133 | 134 | func (cfg Config) Run(global context.Context) error { 135 | ctx, cancel := context.WithCancel(global) 136 | defer cancel() 137 | 138 | srv, err := cfg.Create(global, cfg.DefaultWaitTime) 139 | if err != nil { 140 | return err 141 | } 142 | defer srv.Close() 143 | 144 | httpServer := http.Server{ 145 | Addr: cfg.Bind, 146 | Handler: srv, 147 | } 148 | 149 | done := make(chan struct{}) 150 | 151 | go func() { 152 | defer cancel() 153 | <-ctx.Done() 154 | t, c := context.WithTimeout(context.Background(), cfg.GracefulShutdown) 155 | _ = httpServer.Shutdown(t) 156 | c() 157 | close(done) 158 | }() 159 | 160 | if cfg.TLS.Enable { 161 | err = httpServer.ListenAndServeTLS(cfg.TLS.Cert, cfg.TLS.Key) 162 | } else { 163 | err = httpServer.ListenAndServe() 164 | } 165 | cancel() 166 | <-done 167 | return err 168 | } 169 | 170 | func (cfg Config) installUI(router *gin.Engine, units []server.Unit, workers []*worker.Worker, cronEntries []*server.CronEntry) { 171 | if cfg.DisableUI { 172 | log.Println("ui disabled") 173 | return 174 | } 175 | uiPath := filepath.Join(cfg.UIDirectory, "*.html") 176 | uiGroup := router.Group("/ui/") 177 | if v, err := filepath.Glob(uiPath); err == nil && len(v) > 0 { 178 | cfg.useDirectoryUI(router, uiGroup) 179 | } else { 180 | log.Println("using embedded UI") 181 | cfg.useEmbeddedUI(router, uiGroup) 182 | } 183 | router.GET("/", func(gctx *gin.Context) { 184 | gctx.Redirect(http.StatusTemporaryRedirect, "ui") 185 | }) 186 | ui.Attach(uiGroup, units, workers, cronEntries, cfg.Auth) 187 | } 188 | 189 | func (cfg Config) useDirectoryUI(router *gin.Engine, uiGroup gin.IRouter) { 190 | uiPath := filepath.Join(cfg.UIDirectory, "*.html") 191 | router.SetFuncMap(sprig.HtmlFuncMap()) 192 | router.LoadHTMLGlob(uiPath) 193 | uiGroup.Static("/static/", filepath.Join(cfg.UIDirectory, "static")) 194 | } 195 | 196 | func (cfg Config) useEmbeddedUI(router *gin.Engine, uiGroup gin.IRouter) { 197 | root := template.New("").Funcs(sprig.HtmlFuncMap()) 198 | 199 | for _, src := range embedded_templates.AssetNames() { 200 | _, _ = root.New(src).Parse(string(embedded_templates.MustAsset(src))) 201 | } 202 | router.SetHTMLTemplate(root) 203 | uiGroup.StaticFS("/static/", embedded_static.AssetFile()) 204 | } 205 | 206 | type Server struct { 207 | http.Handler 208 | workers []*worker.Worker 209 | units []server.Unit 210 | cronEntries []*server.CronEntry 211 | cronEngine *cron.Cron 212 | cancel func() 213 | done chan struct{} 214 | err error 215 | } 216 | 217 | func (srv *Server) Units() []server.Unit { return srv.units } 218 | 219 | func (srv *Server) Workers() []*worker.Worker { return srv.workers } 220 | 221 | func (srv *Server) Close() { 222 | for _, wrk := range srv.workers { 223 | wrk.Close() 224 | } 225 | srv.cancel() 226 | <-srv.cronEngine.Stop().Done() 227 | <-srv.done 228 | } 229 | 230 | func (srv *Server) Err() error { 231 | return srv.err 232 | } 233 | 234 | func (srv *Server) run(ctx context.Context) { 235 | srv.cronEngine.Start() 236 | err := server.Run(ctx, srv.workers) 237 | if err != nil { 238 | log.Println("workers stopped:", err) 239 | } 240 | srv.err = err 241 | close(srv.done) 242 | } 243 | -------------------------------------------------------------------------------- /server/server_test.go: -------------------------------------------------------------------------------- 1 | package server_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "net/http/httptest" 11 | "os" 12 | "path/filepath" 13 | "testing" 14 | "time" 15 | 16 | "github.com/stretchr/testify/assert" 17 | 18 | "nano-run/server" 19 | "nano-run/server/runner" 20 | "nano-run/services/meta" 21 | ) 22 | 23 | var tmpDir string 24 | 25 | func TestMain(main *testing.M) { 26 | var err error 27 | tmpDir, err = ioutil.TempDir("", "") 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | code := main.Run() 32 | _ = os.RemoveAll(tmpDir) 33 | os.Exit(code) 34 | } 35 | 36 | func testServer(t *testing.T, cfg runner.Config, units map[string]server.Unit) *runner.Server { 37 | sub, err := ioutil.TempDir(tmpDir, "") 38 | if !assert.NoError(t, err) { 39 | t.Fatal("failed to create temp dir", err) 40 | } 41 | cfg.ConfigDirectory = filepath.Join(sub, "config") 42 | cfg.WorkingDirectory = filepath.Join(sub, "data") 43 | err = cfg.CreateDirs() 44 | if !assert.NoError(t, err) { 45 | t.Fatal("failed to create dirs", err) 46 | } 47 | 48 | for name, unit := range units { 49 | err = unit.SaveFile(filepath.Join(cfg.ConfigDirectory, name+".yaml")) 50 | if !assert.NoError(t, err) { 51 | t.Fatal("failed to create unit", name, ":", err) 52 | } 53 | } 54 | 55 | srv, err := cfg.Create(context.Background(), cfg.DefaultWaitTime) 56 | if !assert.NoError(t, err) { 57 | srv.Close() 58 | t.Fatal("failed to create server") 59 | } 60 | return srv 61 | } 62 | 63 | func Test_create(t *testing.T) { 64 | srv := testServer(t, runner.DefaultConfig(), map[string]server.Unit{ 65 | "hello": { 66 | Command: "echo -n hello world", 67 | }, 68 | }) 69 | defer srv.Close() 70 | 71 | req := httptest.NewRequest(http.MethodPost, "/api/hello/", bytes.NewBufferString("hello world")) 72 | res := httptest.NewRecorder() 73 | srv.ServeHTTP(res, req) 74 | assert.Equal(t, http.StatusSeeOther, res.Code) 75 | assert.NotEmpty(t, res.Header().Get("X-Correlation-Id")) 76 | assert.Equal(t, "/api/hello/"+res.Header().Get("X-Correlation-Id"), res.Header().Get("Location")) 77 | requestID := res.Header().Get("X-Correlation-Id") 78 | 79 | infoURL := res.Header().Get("Location") 80 | t.Log("Location:", infoURL) 81 | req = httptest.NewRequest(http.MethodGet, infoURL, nil) 82 | res = httptest.NewRecorder() 83 | srv.ServeHTTP(res, req) 84 | assert.Equal(t, http.StatusOK, res.Code) 85 | assert.Equal(t, requestID, res.Header().Get("X-Correlation-Id")) 86 | assert.Contains(t, res.Header().Get("Content-Type"), "application/json") 87 | var info meta.Request 88 | err := json.Unmarshal(res.Body.Bytes(), &info) 89 | assert.NoError(t, err) 90 | 91 | // wait for result 92 | var resultLocation string 93 | for { 94 | req = httptest.NewRequest(http.MethodGet, infoURL+"/completed", nil) 95 | res = httptest.NewRecorder() 96 | srv.ServeHTTP(res, req) 97 | if res.Code == http.StatusMovedPermanently { 98 | resultLocation = res.Header().Get("Location") 99 | break 100 | } 101 | if !assert.Equal(t, http.StatusTooEarly, res.Code) { 102 | return 103 | } 104 | time.Sleep(time.Second) 105 | } 106 | 107 | req = httptest.NewRequest(http.MethodGet, resultLocation, nil) 108 | res = httptest.NewRecorder() 109 | srv.ServeHTTP(res, req) 110 | assert.Equal(t, http.StatusOK, res.Code) 111 | assert.Equal(t, "hello world", res.Body.String()) 112 | } 113 | 114 | func Test_retryIfDataReturnedInBinMode(t *testing.T) { 115 | srv := testServer(t, runner.DefaultConfig(), map[string]server.Unit{ 116 | "hello": { 117 | Command: "echo hello world; exit 1", 118 | }, 119 | }) 120 | defer srv.Close() 121 | 122 | req := httptest.NewRequest(http.MethodPost, "/api/hello/", bytes.NewBufferString("hello world")) 123 | res := httptest.NewRecorder() 124 | srv.ServeHTTP(res, req) 125 | assert.Equal(t, http.StatusSeeOther, res.Code) 126 | assert.NotEmpty(t, res.Header().Get("X-Correlation-Id")) 127 | assert.Equal(t, "/api/hello/"+res.Header().Get("X-Correlation-Id"), res.Header().Get("Location")) 128 | location := res.Header().Get("Location") 129 | 130 | // wait for first result 131 | for { 132 | req = httptest.NewRequest(http.MethodGet, location, nil) 133 | res = httptest.NewRecorder() 134 | srv.ServeHTTP(res, req) 135 | if !assert.Equal(t, http.StatusOK, res.Code) { 136 | return 137 | } 138 | var info meta.Request 139 | err := json.Unmarshal(res.Body.Bytes(), &info) 140 | assert.NoError(t, err) 141 | if len(info.Attempts) == 0 { 142 | time.Sleep(time.Second) 143 | continue 144 | } 145 | atp := info.Attempts[0] 146 | assert.Equal(t, http.StatusBadGateway, atp.Code) 147 | assert.Equal(t, "1", atp.Headers.Get("X-Return-Code")) 148 | break 149 | } 150 | 151 | } 152 | 153 | func TestCron(t *testing.T) { 154 | srv := testServer(t, runner.DefaultConfig(), map[string]server.Unit{ 155 | "hello": { 156 | Command: "echo hello world", 157 | Cron: []server.CronSpec{ 158 | {Spec: "@every 1s"}, 159 | }, 160 | }, 161 | }) 162 | defer srv.Close() 163 | time.Sleep(time.Second + 100*time.Millisecond) 164 | 165 | var first *meta.Request 166 | err := srv.Workers()[0].Meta().Iterate(func(id string, record meta.Request) error { 167 | first = &record 168 | return nil 169 | }) 170 | 171 | if !assert.NoError(t, err) { 172 | return 173 | } 174 | 175 | assert.True(t, first.Complete) 176 | assert.Len(t, first.Attempts, 1) 177 | } 178 | 179 | func TestSync(t *testing.T) { 180 | t.Run("wait with default timeout", func(t *testing.T) { 181 | srv := testServer(t, runner.DefaultConfig(), map[string]server.Unit{ 182 | "hello": { 183 | Command: "echo hello world", 184 | }, 185 | }) 186 | defer srv.Close() 187 | 188 | req := httptest.NewRequest(http.MethodPut, "/api/hello/", bytes.NewBufferString("hello world")) 189 | res := httptest.NewRecorder() 190 | srv.ServeHTTP(res, req) 191 | assert.Equal(t, http.StatusSeeOther, res.Code) 192 | }) 193 | t.Run("wait with custom timeout", func(t *testing.T) { 194 | srv := testServer(t, runner.DefaultConfig(), map[string]server.Unit{ 195 | "hello": { 196 | Command: "echo hello world", 197 | }, 198 | }) 199 | defer srv.Close() 200 | 201 | req := httptest.NewRequest(http.MethodPut, "/api/hello/?wait=1h", bytes.NewBufferString("hello world")) 202 | res := httptest.NewRecorder() 203 | srv.ServeHTTP(res, req) 204 | assert.Equal(t, http.StatusSeeOther, res.Code) 205 | }) 206 | t.Run("fail on malformed timeout", func(t *testing.T) { 207 | srv := testServer(t, runner.DefaultConfig(), map[string]server.Unit{ 208 | "hello": { 209 | Command: "echo hello world", 210 | }, 211 | }) 212 | defer srv.Close() 213 | 214 | req := httptest.NewRequest(http.MethodPut, "/api/hello/?wait=10000", bytes.NewBufferString("hello world")) 215 | res := httptest.NewRecorder() 216 | srv.ServeHTTP(res, req) 217 | assert.Equal(t, http.StatusBadRequest, res.Code) 218 | }) 219 | t.Run("fail on invalid timeout", func(t *testing.T) { 220 | srv := testServer(t, runner.DefaultConfig(), map[string]server.Unit{ 221 | "hello": { 222 | Command: "echo hello world", 223 | }, 224 | }) 225 | defer srv.Close() 226 | 227 | req := httptest.NewRequest(http.MethodPut, "/api/hello/?wait=-10s", bytes.NewBufferString("hello world")) 228 | res := httptest.NewRecorder() 229 | srv.ServeHTTP(res, req) 230 | assert.Equal(t, http.StatusBadRequest, res.Code) 231 | }) 232 | } 233 | -------------------------------------------------------------------------------- /server/ui/authorization.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | 7 | "github.com/gin-gonic/gin" 8 | "golang.org/x/oauth2" 9 | ) 10 | 11 | const ( 12 | redirectToCookie = "redirect-to" 13 | ctxLogin = "login" 14 | ctxAuthorized = "authorized" 15 | ) 16 | 17 | type Strategy struct { 18 | Icon string `yaml:"icon"` 19 | Title string `yaml:"title"` 20 | Key string `yaml:"key"` 21 | Secret string `yaml:"secret"` 22 | AuthURL string `yaml:"auth_url"` 23 | TokenURL string `yaml:"token_url"` 24 | ProfileURL string `yaml:"profile_url"` 25 | CallbackURL string `yaml:"callback_url"` 26 | LoginField string `yaml:"login_field"` 27 | Scopes []string `yaml:"scopes"` 28 | } 29 | 30 | func (st *Strategy) Enable(router gin.IRouter, sessionStorage SessionStorage) { 31 | if st == nil { 32 | return 33 | } 34 | ep := &OAuth2{ 35 | Config: oauth2.Config{ 36 | ClientID: st.Key, 37 | ClientSecret: st.Secret, 38 | Endpoint: oauth2.Endpoint{ 39 | AuthURL: st.AuthURL, 40 | TokenURL: st.TokenURL, 41 | }, 42 | RedirectURL: st.CallbackURL, 43 | Scopes: st.Scopes, 44 | }, 45 | ProfileURL: st.ProfileURL, 46 | RedirectTo: "../success", 47 | LoginField: st.LoginField, 48 | } 49 | 50 | ep.Attach(router, sessionStorage) 51 | } 52 | 53 | type Authorization struct { 54 | OAuth2 *Strategy `yaml:"oauth2"` 55 | AllowedUsers []string `yaml:"users"` 56 | } 57 | 58 | func (auth Authorization) Enabled() bool { 59 | return auth.OAuth2 != nil 60 | } 61 | 62 | func (auth Authorization) attach(router gin.IRouter, loginTemplate string, sessionStorage SessionStorage) { 63 | auth.OAuth2.Enable(router.Group("/oauth2/"), sessionStorage) 64 | 65 | router.GET("/logout", func(gctx *gin.Context) { 66 | id, _ := gctx.Cookie(sessionCookie) 67 | sessionStorage.Delete(id) 68 | gctx.Redirect(http.StatusTemporaryRedirect, "../") 69 | }) 70 | router.GET("/success", func(gctx *gin.Context) { 71 | redirectTo, err := gctx.Cookie(redirectToCookie) 72 | if err == nil && redirectTo != "" { 73 | gctx.SetCookie(redirectToCookie, "", -1, "", "", false, true) 74 | gctx.Redirect(http.StatusTemporaryRedirect, redirectTo) 75 | } else { 76 | gctx.Redirect(http.StatusTemporaryRedirect, "../../") 77 | } 78 | }) 79 | router.GET("/", func(gctx *gin.Context) { 80 | var reply struct { 81 | Auth Authorization 82 | } 83 | reply.Auth = auth 84 | gctx.HTML(http.StatusOK, loginTemplate, reply) 85 | }) 86 | } 87 | 88 | func (auth Authorization) restrict(redirectTo func(gctx *gin.Context) string, sessionStorage SessionStorage) gin.HandlerFunc { 89 | if !auth.Enabled() { 90 | return func(gctx *gin.Context) { 91 | gctx.Set(ctxAuthorized, false) 92 | gctx.Set(ctxLogin, "") 93 | gctx.Next() 94 | } 95 | } 96 | return func(gctx *gin.Context) { 97 | sessionID, err := gctx.Cookie(sessionCookie) 98 | session, ok := sessionStorage.Get(sessionID) 99 | if err != nil || !ok || session == nil || !session.Valid() { 100 | gctx.SetCookie(redirectToCookie, gctx.Request.RequestURI, 3600, "", "", false, true) 101 | gctx.Redirect(http.StatusTemporaryRedirect, redirectTo(gctx)) 102 | gctx.Abort() 103 | return 104 | } 105 | login := session.Login() 106 | 107 | found := len(auth.AllowedUsers) == 0 108 | for _, u := range auth.AllowedUsers { 109 | if u == login { 110 | found = true 111 | break 112 | } 113 | } 114 | if !found { 115 | log.Println("user", login, "not allowed") 116 | gctx.Redirect(http.StatusTemporaryRedirect, redirectTo(gctx)) 117 | gctx.Abort() 118 | return 119 | } 120 | gctx.Set(ctxAuthorized, true) 121 | gctx.Set(ctxLogin, login) 122 | gctx.Next() 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /server/ui/oauth.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | 11 | "github.com/gin-gonic/gin" 12 | "github.com/google/uuid" 13 | "golang.org/x/oauth2" 14 | ) 15 | 16 | type OAuth2 struct { 17 | Config oauth2.Config 18 | ProfileURL string 19 | RedirectTo string 20 | LoginField string 21 | } 22 | 23 | func (cfg OAuth2) Attach(router gin.IRouter, storage SessionStorage) { 24 | router.GET("/login", func(gctx *gin.Context) { 25 | state := uuid.New().String() 26 | gctx.SetCookie("oauth", state, 3600, "", "", false, true) 27 | u := cfg.Config.AuthCodeURL(state) 28 | gctx.Redirect(http.StatusTemporaryRedirect, u) 29 | }) 30 | router.GET("/callback", func(gctx *gin.Context) { 31 | savedState, _ := gctx.Cookie("oauth") 32 | if savedState != gctx.Query("state") { 33 | gctx.AbortWithStatus(http.StatusForbidden) 34 | return 35 | } 36 | gctx.SetCookie("oauth", "", -1, "", "", false, true) 37 | 38 | token, err := cfg.Config.Exchange(gctx.Request.Context(), gctx.Query("code")) 39 | if err != nil { 40 | _ = gctx.AbortWithError(http.StatusForbidden, err) 41 | return 42 | } 43 | 44 | if !token.Valid() { 45 | gctx.AbortWithStatus(http.StatusForbidden) 46 | return 47 | } 48 | 49 | sessionID := uuid.New().String() 50 | session := newOAuthSession(token) 51 | if cfg.ProfileURL != "" { 52 | err = session.fetchLogin(gctx.Request.Context(), cfg.ProfileURL, cfg.LoginField) 53 | if err != nil { 54 | _ = gctx.AbortWithError(http.StatusForbidden, err) 55 | return 56 | } 57 | } 58 | 59 | gctx.SetCookie(sessionCookie, sessionID, 0, "", "", false, true) 60 | 61 | storage.Save(sessionID, session) 62 | log.Println("user", session.Login(), "authorized via oauth2") 63 | 64 | gctx.Redirect(http.StatusTemporaryRedirect, cfg.RedirectTo) 65 | }) 66 | } 67 | 68 | func newOAuthSession(token *oauth2.Token) *oauthSession { 69 | return &oauthSession{ 70 | token: oauth2.StaticTokenSource(token), 71 | } 72 | } 73 | 74 | type oauthSession struct { 75 | login string 76 | token oauth2.TokenSource 77 | } 78 | 79 | func (ss *oauthSession) Login() string { 80 | return ss.login 81 | } 82 | 83 | func (ss *oauthSession) Valid() bool { 84 | token, err := ss.token.Token() 85 | if err != nil { 86 | log.Println(err) 87 | return false 88 | } 89 | return token.Valid() 90 | } 91 | 92 | func (ss *oauthSession) GetJSON(ctx context.Context, url string, response interface{}) error { 93 | t, err := ss.token.Token() 94 | if err != nil { 95 | return err 96 | } 97 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 98 | if err != nil { 99 | return err 100 | } 101 | t.SetAuthHeader(req) 102 | res, err := http.DefaultClient.Do(req) 103 | if err != nil { 104 | return err 105 | } 106 | defer res.Body.Close() 107 | if res.StatusCode != http.StatusOK { 108 | return errors.New(res.Status) 109 | } 110 | return json.NewDecoder(res.Body).Decode(response) 111 | } 112 | 113 | func (ss *oauthSession) fetchLogin(ctx context.Context, url string, field string) error { 114 | var profile = make(map[string]interface{}) 115 | err := ss.GetJSON(ctx, url, &profile) 116 | if err != nil { 117 | return err 118 | } 119 | log.Printf("profile: %+v", profile) 120 | l, ok := profile[field] 121 | if !ok { 122 | return fmt.Errorf("not field %s in response", field) 123 | } 124 | s, ok := l.(string) 125 | if !ok { 126 | return fmt.Errorf("field %s is not string", field) 127 | } 128 | ss.login = s 129 | return nil 130 | } 131 | -------------------------------------------------------------------------------- /server/ui/router.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "log" 7 | "net/http" 8 | "path/filepath" 9 | "sort" 10 | "strconv" 11 | "time" 12 | 13 | "github.com/Masterminds/sprig" 14 | "github.com/gin-gonic/gin" 15 | 16 | "nano-run/server" 17 | "nano-run/services/meta" 18 | "nano-run/worker" 19 | ) 20 | 21 | func Expose(units []server.Unit, workers []*worker.Worker, cronEntries []*server.CronEntry, uiDir string, auth Authorization) http.Handler { 22 | router := gin.New() 23 | router.SetFuncMap(sprig.HtmlFuncMap()) 24 | router.LoadHTMLGlob(filepath.Join(uiDir, "*.html")) 25 | Attach(router, units, workers, cronEntries, auth) 26 | return router 27 | } 28 | 29 | func Attach(router gin.IRouter, units []server.Unit, workers []*worker.Worker, cronEntries []*server.CronEntry, auth Authorization) { 30 | ui := &uiRouter{ 31 | units: make(map[string]unitInfo), 32 | } 33 | 34 | var offset int 35 | for i := range units { 36 | u := units[i] 37 | w := workers[i] 38 | var ce []*server.CronEntry 39 | 40 | var last int 41 | for last = offset; last < len(cronEntries); last++ { 42 | if cronEntries[last].Worker != w { 43 | break 44 | } 45 | } 46 | ce = cronEntries[offset:last] 47 | offset = last 48 | 49 | ui.units[u.Name()] = unitInfo{ 50 | Unit: u, 51 | Worker: w, 52 | CronEntries: ce, 53 | } 54 | } 55 | sessions := &memorySessions{} 56 | if r, ok := router.(*gin.RouterGroup); ok { 57 | router.Use(func(gctx *gin.Context) { 58 | gctx.Set("base", r.BasePath()) 59 | gctx.Next() 60 | }) 61 | } 62 | router.GET("", func(gctx *gin.Context) { 63 | gctx.Redirect(http.StatusTemporaryRedirect, "unit/") 64 | }) 65 | auth.attach(router.Group("/auth"), "login.html", sessions) 66 | 67 | guard := auth.restrict(func(gctx *gin.Context) string { 68 | b := base(gctx) 69 | return b.Rel("/auth/") 70 | }, sessions) 71 | 72 | unitsRoutes := router.Group("/unit/").Use(guard) 73 | 74 | unitsRoutes.GET("/", ui.listUnits) 75 | unitsRoutes.GET("/:name/", ui.unitInfo) 76 | unitsRoutes.POST("/:name/", ui.unitInvoke) 77 | unitsRoutes.GET("/:name/history", ui.unitHistory) 78 | unitsRoutes.GET("/:name/request/:request/", ui.unitRequestInfo) 79 | unitsRoutes.POST("/:name/request/:request/", ui.unitRequestRetry) 80 | unitsRoutes.GET("/:name/request/:request/payload", ui.unitRequestPayload) 81 | unitsRoutes.GET("/:name/request/:request/attempt/:attempt/", ui.unitRequestAttemptInfo) 82 | unitsRoutes.GET("/:name/request/:request/attempt/:attempt/result", ui.unitRequestAttemptResult) 83 | unitsRoutes.GET("/:name/cron/:index", ui.unitCronInfo) 84 | 85 | cronRoutes := router.Group("/cron/").Use(guard) 86 | cronRoutes.GET("/", ui.listCron) 87 | } 88 | 89 | type uiRouter struct { 90 | units map[string]unitInfo 91 | } 92 | 93 | func (ui *uiRouter) unitRequestAttemptResult(gctx *gin.Context) { 94 | name := gctx.Param("name") 95 | info, ok := ui.units[name] 96 | if !ok { 97 | gctx.AbortWithStatus(http.StatusNotFound) 98 | return 99 | } 100 | 101 | attemptID := gctx.Param("attempt") 102 | 103 | f, err := info.Worker.Blobs().Get(attemptID) 104 | if err != nil { 105 | _ = gctx.AbortWithError(http.StatusNotFound, err) 106 | return 107 | } 108 | defer f.Close() 109 | gctx.Status(http.StatusOK) 110 | _, _ = io.Copy(gctx.Writer, f) 111 | } 112 | 113 | type attemptInfo struct { 114 | requestInfo 115 | AttemptID string 116 | Attempt meta.Attempt 117 | } 118 | 119 | func (ui *uiRouter) unitRequestAttemptInfo(gctx *gin.Context) { 120 | name := gctx.Param("name") 121 | info, ok := ui.units[name] 122 | if !ok { 123 | gctx.AbortWithStatus(http.StatusNotFound) 124 | return 125 | } 126 | requestID := gctx.Param("request") 127 | request, err := info.Worker.Meta().Get(requestID) 128 | if err != nil { 129 | _ = gctx.AbortWithError(http.StatusNotFound, err) 130 | return 131 | } 132 | 133 | attemptID := gctx.Param("attempt") 134 | 135 | var found bool 136 | var attempt meta.Attempt 137 | for _, atp := range request.Attempts { 138 | if atp.ID == attemptID { 139 | attempt = atp 140 | found = true 141 | break 142 | } 143 | } 144 | if !found { 145 | gctx.AbortWithStatus(http.StatusNotFound) 146 | return 147 | } 148 | 149 | gctx.HTML(http.StatusOK, "unit-request-attempt-info.html", attemptInfo{ 150 | requestInfo: requestInfo{ 151 | baseResponse: base(gctx), 152 | unitInfo: info, 153 | Request: request, 154 | RequestID: requestID, 155 | }, 156 | AttemptID: attemptID, 157 | Attempt: attempt, 158 | }) 159 | } 160 | 161 | type requestInfo struct { 162 | unitInfo 163 | baseResponse 164 | Request *meta.Request 165 | RequestID string 166 | } 167 | 168 | func (ui *uiRouter) unitRequestInfo(gctx *gin.Context) { 169 | name := gctx.Param("name") 170 | info, ok := ui.units[name] 171 | if !ok { 172 | gctx.AbortWithStatus(http.StatusNotFound) 173 | return 174 | } 175 | requestID := gctx.Param("request") 176 | request, err := info.Worker.Meta().Get(requestID) 177 | if err != nil { 178 | _ = gctx.AbortWithError(http.StatusNotFound, err) 179 | return 180 | } 181 | 182 | gctx.HTML(http.StatusOK, "unit-request-info.html", requestInfo{ 183 | baseResponse: base(gctx), 184 | unitInfo: info, 185 | Request: request, 186 | RequestID: requestID, 187 | }) 188 | } 189 | 190 | func (ui *uiRouter) unitRequestRetry(gctx *gin.Context) { 191 | name := gctx.Param("name") 192 | item, ok := ui.units[name] 193 | if !ok { 194 | gctx.AbortWithStatus(http.StatusNotFound) 195 | return 196 | } 197 | requestID := gctx.Param("request") 198 | 199 | id, err := item.Worker.Retry(gctx.Request.Context(), requestID) 200 | if err != nil { 201 | log.Println("failed to retry:", err) 202 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 203 | return 204 | } 205 | 206 | gctx.Redirect(http.StatusSeeOther, base(gctx).Rel("/unit", name, "request", id)) 207 | } 208 | 209 | func (ui *uiRouter) unitRequestPayload(gctx *gin.Context) { 210 | name := gctx.Param("name") 211 | item, ok := ui.units[name] 212 | if !ok { 213 | gctx.AbortWithStatus(http.StatusNotFound) 214 | return 215 | } 216 | requestID := gctx.Param("request") 217 | info, err := item.Worker.Meta().Get(requestID) 218 | if !ok { 219 | _ = gctx.AbortWithError(http.StatusNotFound, err) 220 | return 221 | } 222 | gctx.Header("Last-Modified", info.CreatedAt.Format(time.RFC850)) 223 | f, err := item.Worker.Blobs().Get(requestID) 224 | if err != nil { 225 | log.Println("failed to get data:", err) 226 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 227 | return 228 | } 229 | defer f.Close() 230 | 231 | gctx.Header("X-Method", info.Method) 232 | gctx.Header("X-Request-Uri", info.URI) 233 | 234 | for k, v := range info.Headers { 235 | gctx.Request.Header[k] = v 236 | } 237 | gctx.Status(http.StatusOK) 238 | _, _ = io.Copy(gctx.Writer, f) 239 | } 240 | 241 | func (ui *uiRouter) unitInfo(gctx *gin.Context) { 242 | type viewUnit struct { 243 | unitInfo 244 | baseResponse 245 | } 246 | name := gctx.Param("name") 247 | info, ok := ui.units[name] 248 | if !ok { 249 | gctx.AbortWithStatus(http.StatusNotFound) 250 | return 251 | } 252 | gctx.HTML(http.StatusOK, "unit-info.html", viewUnit{ 253 | unitInfo: info, 254 | baseResponse: base(gctx), 255 | }) 256 | } 257 | 258 | func (ui *uiRouter) unitInvoke(gctx *gin.Context) { 259 | name := gctx.Param("name") 260 | info, ok := ui.units[name] 261 | if !ok { 262 | gctx.AbortWithStatus(http.StatusNotFound) 263 | return 264 | } 265 | data := gctx.PostForm("body") 266 | req, err := http.NewRequestWithContext(gctx.Request.Context(), http.MethodPost, "/", bytes.NewBufferString(data)) 267 | if err != nil { 268 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 269 | return 270 | } 271 | 272 | id, err := info.Worker.Enqueue(req) 273 | if err != nil { 274 | _ = gctx.AbortWithError(http.StatusInternalServerError, err) 275 | return 276 | } 277 | gctx.Redirect(http.StatusSeeOther, base(gctx).Rel("/unit", name, "request", id)) 278 | } 279 | 280 | func (ui *uiRouter) listUnits(gctx *gin.Context) { 281 | var reply struct { 282 | baseResponse 283 | Units []server.Unit 284 | } 285 | 286 | var units = make([]server.Unit, 0, len(ui.units)) 287 | for _, info := range ui.units { 288 | units = append(units, info.Unit) 289 | } 290 | sort.Slice(units, func(i, j int) bool { 291 | return units[i].Name() < units[j].Name() 292 | }) 293 | reply.baseResponse = base(gctx) 294 | reply.Units = units 295 | gctx.HTML(http.StatusOK, "units-list.html", reply) 296 | } 297 | 298 | func (ui *uiRouter) listCron(gctx *gin.Context) { 299 | type uiEntry struct { 300 | Index int 301 | Entry *server.CronEntry 302 | } 303 | 304 | var reply struct { 305 | baseResponse 306 | Entries []uiEntry 307 | } 308 | 309 | var specs = make([]uiEntry, 0, len(ui.units)) 310 | for _, info := range ui.units { 311 | for i, spec := range info.CronEntries { 312 | specs = append(specs, uiEntry{ 313 | Index: i, 314 | Entry: spec, 315 | }) 316 | } 317 | } 318 | sort.Slice(specs, func(i, j int) bool { 319 | if specs[i].Entry.Config.Name() < specs[j].Entry.Config.Name() { 320 | return true 321 | } else if specs[i].Entry.Config.Name() == specs[j].Entry.Config.Name() && specs[i].Index < specs[j].Index { 322 | return true 323 | } 324 | return false 325 | }) 326 | reply.baseResponse = base(gctx) 327 | reply.Entries = specs 328 | gctx.HTML(http.StatusOK, "cron-list.html", reply) 329 | } 330 | 331 | func (ui *uiRouter) unitHistory(gctx *gin.Context) { 332 | type viewUnit struct { 333 | unitInfo 334 | baseResponse 335 | } 336 | name := gctx.Param("name") 337 | info, ok := ui.units[name] 338 | if !ok { 339 | gctx.AbortWithStatus(http.StatusNotFound) 340 | return 341 | } 342 | gctx.HTML(http.StatusOK, "unit-history.html", viewUnit{ 343 | unitInfo: info, 344 | baseResponse: base(gctx), 345 | }) 346 | } 347 | 348 | func (ui *uiRouter) unitCronInfo(gctx *gin.Context) { 349 | type viewUnit struct { 350 | unitInfo 351 | baseResponse 352 | Cron *server.CronEntry 353 | Label string 354 | } 355 | name := gctx.Param("name") 356 | info, ok := ui.units[name] 357 | if !ok { 358 | gctx.AbortWithStatus(http.StatusNotFound) 359 | return 360 | } 361 | 362 | strIndex := gctx.Param("index") 363 | index, err := strconv.Atoi(strIndex) 364 | 365 | if err != nil || index < 0 || index >= len(info.CronEntries) { 366 | gctx.AbortWithStatus(http.StatusNotFound) 367 | return 368 | } 369 | entry := info.CronEntries[index] 370 | gctx.HTML(http.StatusOK, "unit-cron-info.html", viewUnit{ 371 | unitInfo: info, 372 | baseResponse: base(gctx), 373 | Cron: entry, 374 | Label: entry.Spec.Label(strconv.Itoa(index)), 375 | }) 376 | } 377 | -------------------------------------------------------------------------------- /server/ui/sessions.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import "sync" 4 | 5 | const sessionCookie = "session-id" 6 | 7 | type Session interface { 8 | Login() string 9 | Valid() bool 10 | } 11 | 12 | type SessionStorage interface { 13 | Save(sessionID string, session Session) 14 | Get(sessionID string) (Session, bool) 15 | Delete(sessionID string) 16 | } 17 | 18 | type memorySessions struct { 19 | store sync.Map 20 | } 21 | 22 | func (ms *memorySessions) Save(sessionID string, session Session) { 23 | ms.store.Store(sessionID, session) 24 | } 25 | 26 | func (ms *memorySessions) Get(sessionID string) (Session, bool) { 27 | v, ok := ms.store.Load(sessionID) 28 | if !ok { 29 | return nil, ok 30 | } 31 | s, ok := v.(Session) 32 | return s, ok 33 | } 34 | 35 | func (ms *memorySessions) Delete(sessionID string) { 36 | ms.store.Delete(sessionID) 37 | } 38 | -------------------------------------------------------------------------------- /server/ui/unit_info.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "os" 5 | 6 | "nano-run/server" 7 | "nano-run/services/meta" 8 | "nano-run/worker" 9 | ) 10 | 11 | type unitInfo struct { 12 | Unit server.Unit 13 | Worker *worker.Worker 14 | CronEntries []*server.CronEntry 15 | } 16 | 17 | type historyRecord struct { 18 | ID string 19 | Meta meta.Request 20 | } 21 | 22 | func (info unitInfo) History(max int) ([]historyRecord, error) { 23 | var res []historyRecord 24 | 25 | err := info.Worker.Meta().Iterate(func(id string, record meta.Request) error { 26 | if len(res) >= max { 27 | return os.ErrClosed 28 | } 29 | res = append(res, historyRecord{ 30 | ID: id, 31 | Meta: record, 32 | }) 33 | return nil 34 | }) 35 | if err == os.ErrClosed || err == nil { 36 | return res, nil 37 | } 38 | return nil, err 39 | } 40 | -------------------------------------------------------------------------------- /server/ui/utils.go: -------------------------------------------------------------------------------- 1 | package ui 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "strings" 7 | 8 | "github.com/gin-gonic/gin" 9 | ) 10 | 11 | type baseResponse struct { 12 | Context *gin.Context 13 | Login string 14 | Authorized bool 15 | BaseURL string 16 | } 17 | 18 | func (br baseResponse) Rel(path string, tail ...string) string { 19 | var chunks = append([]string{path}, tail...) 20 | path = strings.Join(chunks, "/") 21 | if len(path) == 0 || path[0] != '/' { 22 | return path 23 | } 24 | toRoot := strings.Repeat("../", strings.Count(br.Context.Request.RequestURI, "/")) 25 | if len(toRoot) > 0 { 26 | toRoot = toRoot[:len(toRoot)-1] 27 | } 28 | return toRoot + br.Context.GetString("base") + path[1:] 29 | } 30 | 31 | func base(gctx *gin.Context) baseResponse { 32 | return baseResponse{ 33 | Authorized: gctx.GetBool(ctxAuthorized), 34 | Context: gctx, 35 | Login: gctx.GetString(ctxLogin), 36 | BaseURL: getProto(gctx.Request) + "://" + gctx.Request.Host, 37 | } 38 | } 39 | 40 | func getProto(req *http.Request) string { 41 | return extractScheme(req.Header.Get("Origin"), extractScheme(req.Header.Get("Referer"), "https")) 42 | } 43 | 44 | func extractScheme(guess, fallback string) string { 45 | if guess == "" { 46 | return fallback 47 | } 48 | u, err := url.Parse(guess) 49 | if err != nil { 50 | return fallback 51 | } 52 | return u.Scheme 53 | } 54 | -------------------------------------------------------------------------------- /server/unit.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "log" 10 | "net/http" 11 | "net/http/cgi" //nolint:gosec 12 | "net/http/httputil" 13 | "net/url" 14 | "os" 15 | "path/filepath" 16 | "runtime" 17 | "strconv" 18 | "strings" 19 | "sync" 20 | "time" 21 | 22 | "github.com/gin-gonic/gin" 23 | "gopkg.in/yaml.v2" 24 | 25 | "nano-run/server/api" 26 | "nano-run/worker" 27 | ) 28 | 29 | type Unit struct { 30 | Private bool `yaml:"private,omitempty"` // private unit - do not expose over API, could useful for cron-only tasks 31 | Interval time.Duration `yaml:"interval,omitempty"` // interval between attempts 32 | Attempts int `yaml:"attempts,omitempty"` // maximum number of attempts 33 | Workers int `yaml:"workers,omitempty"` // concurrency level - number of parallel requests 34 | Mode string `yaml:"mode,omitempty"` // execution mode: bin, cgi or proxy 35 | WorkDir string `yaml:"workdir,omitempty"` // working directory for the worker. if empty - temporary one will generated automatically 36 | Command string `yaml:"command"` // command in a shell to execute 37 | User string `yaml:"user,omitempty"` // user name as process owner (only for bin mode) 38 | Timeout time.Duration `yaml:"timeout,omitempty"` // maximum execution timeout (enabled only for bin mode and only if positive) 39 | GracefulTimeout time.Duration `yaml:"graceful_timeout,omitempty"` // maximum execution timeout after which SIGINT will be sent (enabled only for bin mode and only if positive) 40 | Shell string `yaml:"shell,omitempty"` // shell to execute command in bin mode (default - /bin/sh) 41 | Environment map[string]string `yaml:"environment,omitempty"` // custom environment for executable (in addition to system) 42 | MaxRequest int64 `yaml:"max_request,omitempty"` // optional maximum HTTP body size (enabled if positive) 43 | Authorization struct { 44 | JWT struct { 45 | Enable bool `yaml:"enable"` // enable JWT verification 46 | JWT `yaml:",inline"` 47 | } `yaml:"jwt,omitempty"` // HMAC256 JWT verification with shared secret 48 | 49 | QueryToken struct { 50 | Enable bool `yaml:"enable"` // enable query-based token access 51 | QueryToken `yaml:",inline"` 52 | } `yaml:"query_token,omitempty"` // plain API tokens in request query params 53 | 54 | HeaderToken struct { 55 | Enable bool `yaml:"enable"` // enable header-based token access 56 | HeaderToken `yaml:",inline"` 57 | } `yaml:"header_token,omitempty"` // plain API tokens in request header 58 | 59 | Basic struct { 60 | Enable bool `yaml:"enable"` // enable basic verification 61 | Basic `yaml:",inline"` 62 | } `yaml:"basic,omitempty"` // basic authorization 63 | } `yaml:"authorization,omitempty"` 64 | Cron []CronSpec `yaml:"cron,omitempty"` // cron-tab like definition (see CronSpec) 65 | name string 66 | } 67 | 68 | const ( 69 | defaultRequestSize = 1 * 1024 * 1024 // 1MB 70 | defaultAttempts = 3 71 | defaultInterval = 5 * time.Second 72 | defaultWorkers = 1 73 | defaultShell = "/bin/sh" 74 | defaultMode = "bin" 75 | defaultCommand = "echo hello world" 76 | defaultName = "main" 77 | ) 78 | 79 | func DefaultUnit() Unit { 80 | return Unit{ 81 | Interval: defaultInterval, 82 | Attempts: defaultAttempts, 83 | Workers: defaultWorkers, 84 | MaxRequest: defaultRequestSize, 85 | Shell: defaultShell, 86 | Mode: defaultMode, 87 | Command: defaultCommand, 88 | name: defaultName, 89 | } 90 | } 91 | 92 | func (cfg Unit) Validate() error { 93 | var checks []string 94 | if cfg.Interval < 0 { 95 | checks = append(checks, "negative interval") 96 | } 97 | if cfg.Attempts < 0 { 98 | checks = append(checks, "negative attempts") 99 | } 100 | if cfg.Workers < 0 { 101 | checks = append(checks, "negative workers amount") 102 | } 103 | if !(cfg.Mode == "bin" || cfg.Mode == "cgi" || cfg.Mode == "proxy") { 104 | checks = append(checks, "unknown mode "+cfg.Mode) 105 | } 106 | for i, spec := range cfg.Cron { 107 | err := spec.Validate() 108 | if err != nil { 109 | checks = append(checks, "cron "+spec.Label(strconv.Itoa(i))+": "+err.Error()) 110 | } 111 | } 112 | if len(checks) == 0 { 113 | return nil 114 | } 115 | return errors.New(strings.Join(checks, ", ")) 116 | } 117 | 118 | func (cfg Unit) SaveFile(file string) error { 119 | data, err := yaml.Marshal(cfg) 120 | if err != nil { 121 | return err 122 | } 123 | return ioutil.WriteFile(file, data, 0600) 124 | } 125 | 126 | func (cfg Unit) Name() string { return cfg.name } 127 | 128 | func (cfg Unit) Path() string { return "/" + cfg.name + "/" } 129 | 130 | func (cfg Unit) Secured() bool { 131 | return cfg.Authorization.Basic.Enable || 132 | cfg.Authorization.HeaderToken.Enable || 133 | cfg.Authorization.QueryToken.Enable || 134 | cfg.Authorization.JWT.Enable 135 | } 136 | 137 | func Units(configsDir string) ([]Unit, error) { 138 | var configs []Unit 139 | err := filepath.Walk(configsDir, func(path string, info os.FileInfo, err error) error { 140 | if err != nil { 141 | return err 142 | } 143 | if info.IsDir() { 144 | return nil 145 | } 146 | name := info.Name() 147 | if !(strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml")) { 148 | return nil 149 | } 150 | unitName := strings.ReplaceAll(strings.Trim(path[len(configsDir):strings.LastIndex(path, ".")], "/\\"), string(filepath.Separator), "-") 151 | cfg := DefaultUnit() 152 | cfg.name = unitName 153 | data, err := ioutil.ReadFile(path) 154 | if err != nil { 155 | return err 156 | } 157 | err = yaml.Unmarshal(data, &cfg) 158 | if err != nil { 159 | return err 160 | } 161 | configs = append(configs, cfg) 162 | return nil 163 | }) 164 | return configs, err 165 | } 166 | 167 | func Workers(workdir string, configurations []Unit) ([]*worker.Worker, error) { 168 | var ans []*worker.Worker 169 | for _, cfg := range configurations { 170 | log.Println("validating", cfg.name) 171 | if err := cfg.Validate(); err != nil { 172 | return nil, fmt.Errorf("configuration invalid for %s: %w", cfg.name, err) 173 | } 174 | if cfg.Workers == 0 { 175 | cfg.Workers = runtime.NumCPU() 176 | } 177 | wrk, err := cfg.worker(workdir) 178 | if err != nil { 179 | for _, w := range ans { 180 | w.Close() 181 | } 182 | return nil, err 183 | } 184 | ans = append(ans, wrk) 185 | } 186 | return ans, nil 187 | } 188 | 189 | func Handler(units []Unit, workers []*worker.Worker, defaultWaitTime time.Duration) http.Handler { 190 | router := gin.New() 191 | Attach(router, units, workers, defaultWaitTime) 192 | return router 193 | } 194 | 195 | func Attach(router gin.IRouter, units []Unit, workers []*worker.Worker, defaultWaitTime time.Duration) { 196 | for i, unit := range units { 197 | if !unit.Private { 198 | group := router.Group(unit.Path()) 199 | group.Use(unit.enableAuthorization()) 200 | api.Expose(group, workers[i], defaultWaitTime) 201 | } else { 202 | log.Println("do not expose unit", unit.Name(), "because it's private") 203 | } 204 | } 205 | } 206 | 207 | func Run(global context.Context, workers []*worker.Worker) error { 208 | if len(workers) == 0 { 209 | <-global.Done() 210 | return global.Err() 211 | } 212 | 213 | ctx, cancel := context.WithCancel(global) 214 | defer cancel() 215 | var wg sync.WaitGroup 216 | 217 | for _, wrk := range workers { 218 | wg.Add(1) 219 | go func(wrk *worker.Worker) { 220 | err := wrk.Run(ctx) 221 | if err != nil { 222 | log.Println("failed:", err) 223 | } 224 | wg.Done() 225 | }(wrk) 226 | } 227 | 228 | wg.Wait() 229 | return ctx.Err() 230 | } 231 | 232 | func (cfg Unit) worker(root string) (*worker.Worker, error) { 233 | handler, err := cfg.handler() 234 | if err != nil { 235 | return nil, err 236 | } 237 | workdir := filepath.Join(root, cfg.name) 238 | wrk, err := worker.Default(workdir) 239 | if err != nil { 240 | return nil, err 241 | } 242 | wrk = wrk.Attempts(cfg.Attempts).Interval(cfg.Interval).Concurrency(cfg.Workers).Handler(handler) 243 | return wrk, nil 244 | } 245 | 246 | func (cfg Unit) handler() (http.Handler, error) { 247 | handler, err := cfg.createRunner() 248 | if err != nil { 249 | return nil, err 250 | } 251 | if cfg.MaxRequest > 0 { 252 | handler = limitRequest(cfg.MaxRequest, handler) 253 | } 254 | //TODO: add authorization 255 | return handler, nil 256 | } 257 | 258 | func (cfg Unit) createRunner() (http.Handler, error) { 259 | switch cfg.Mode { 260 | case "bin": 261 | return &binHandler{ 262 | user: cfg.User, 263 | command: cfg.Command, 264 | workDir: cfg.WorkDir, 265 | shell: cfg.Shell, 266 | timeout: cfg.Timeout, 267 | gracefulTimeout: cfg.GracefulTimeout, 268 | environment: append(os.Environ(), makeEnvList(cfg.Environment)...), 269 | }, nil 270 | case "cgi": 271 | return &cgi.Handler{ 272 | Path: cfg.Shell, 273 | Dir: cfg.WorkDir, 274 | Env: append(os.Environ(), makeEnvList(cfg.Environment)...), 275 | Logger: log.New(os.Stderr, "[cgi] ", log.LstdFlags), 276 | Args: []string{"-c", cfg.Command}, 277 | Stderr: os.Stderr, 278 | }, nil 279 | case "proxy": 280 | // proxy to static URL 281 | u, err := url.Parse(cfg.Command) 282 | if err != nil { 283 | return nil, err 284 | } 285 | return httputil.NewSingleHostReverseProxy(u), nil 286 | default: 287 | return nil, fmt.Errorf("unknown mode %s", cfg.Mode) 288 | } 289 | } 290 | 291 | func makeEnvList(content map[string]string) []string { 292 | var ans = make([]string, 0, len(content)) 293 | for k, v := range content { 294 | ans = append(ans, k+"="+v) 295 | } 296 | return ans 297 | } 298 | 299 | func limitRequest(maxSize int64, handler http.Handler) http.Handler { 300 | return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { 301 | body := request.Body 302 | defer body.Close() 303 | if request.ContentLength > maxSize { 304 | http.Error(writer, "too big request", http.StatusBadRequest) 305 | return 306 | } 307 | 308 | limiter := io.LimitReader(request.Body, maxSize) 309 | request.Body = ioutil.NopCloser(limiter) 310 | handler.ServeHTTP(writer, request) 311 | }) 312 | } 313 | -------------------------------------------------------------------------------- /services/blob/fsblob/impl.go: -------------------------------------------------------------------------------- 1 | package fsblob 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | 9 | "nano-run/services/blob" 10 | ) 11 | 12 | // Dummy file storage: large storage based on file system. 13 | // ID - just name of file, but content will be written atomically. 14 | func New(rootDir string) blob.Blob { 15 | return &fsBlob{ 16 | rootDir: rootDir, 17 | checker: func(id string) bool { 18 | return true 19 | }, 20 | } 21 | } 22 | 23 | func NewCheck(rootDir string, checkFn func(string) bool) blob.Blob { 24 | return &fsBlob{ 25 | rootDir: rootDir, 26 | checker: checkFn, 27 | } 28 | } 29 | 30 | type fsBlob struct { 31 | rootDir string 32 | checker func(id string) bool 33 | } 34 | 35 | func (k *fsBlob) Push(id string, handler func(out io.Writer) error) error { 36 | if !k.checker(id) { 37 | return fmt.Errorf("push: invalid id %s", id) 38 | } 39 | dir := filepath.Join(k.rootDir, id[0:1]) 40 | err := os.MkdirAll(dir, 0755) 41 | if err != nil { 42 | return err 43 | } 44 | tempFile := filepath.Join(dir, id+".temp") 45 | destFile := filepath.Join(dir, id) 46 | 47 | tempF, err := os.Create(tempFile) 48 | if err != nil { 49 | return fmt.Errorf("fsblob: put data: create temp file: %w", err) 50 | } 51 | err = handler(tempF) 52 | flushErr := tempF.Close() 53 | if err != nil { 54 | _ = os.Remove(tempFile) 55 | return err 56 | } 57 | if flushErr != nil { 58 | _ = os.Remove(tempFile) 59 | return fmt.Errorf("fsblob: put data: flush content to temp file: %w", err) 60 | } 61 | 62 | err = os.Rename(tempFile, destFile) 63 | if err != nil { 64 | return fmt.Errorf("fsblob: put data: commit file: %w", err) 65 | } 66 | return nil 67 | } 68 | 69 | func (k *fsBlob) Get(id string) (io.ReadCloser, error) { 70 | if !k.checker(id) { 71 | return nil, fmt.Errorf("get: invalid id %s", id) 72 | } 73 | dir := filepath.Join(k.rootDir, id[0:1]) 74 | destFile := filepath.Join(dir, id) 75 | f, err := os.Open(destFile) 76 | if err == nil { 77 | return f, nil 78 | } 79 | return nil, fmt.Errorf("fsblob: get file: %w", err) 80 | } 81 | -------------------------------------------------------------------------------- /services/blob/interface.go: -------------------------------------------------------------------------------- 1 | package blob 2 | 3 | import "io" 4 | 5 | // Large (more then memory) content storage. 6 | type Blob interface { 7 | // Push content to the storage using provided writer. 8 | Push(id string, handler func(out io.Writer) error) error 9 | // Get content from the storage. 10 | Get(id string) (io.ReadCloser, error) 11 | } 12 | -------------------------------------------------------------------------------- /services/meta/interface.go: -------------------------------------------------------------------------------- 1 | package meta 2 | 3 | import ( 4 | "net/http" 5 | "time" 6 | ) 7 | 8 | type Meta interface { 9 | Get(requestID string) (*Request, error) 10 | CreateRequest(requestID string, headers http.Header, uri string, method string) error 11 | AddAttempt(requestID, attemptID string, header AttemptHeader) (*Request, error) 12 | Complete(requestID string) error 13 | Iterate(handler func(id string, record Request) error) error 14 | } 15 | 16 | type AttemptHeader struct { 17 | Code int `json:"code"` 18 | Headers http.Header `json:"headers"` 19 | StartedAt time.Time `json:"started_at"` 20 | } 21 | 22 | type Attempt struct { 23 | AttemptHeader 24 | ID string `json:"id"` 25 | CreatedAt time.Time `json:"created_at"` 26 | } 27 | 28 | type Request struct { 29 | CreatedAt time.Time `json:"created_at"` 30 | CompleteAt time.Time `json:"complete_at,omitempty"` 31 | Attempts []Attempt `json:"attempts"` 32 | Headers http.Header `json:"headers"` 33 | URI string `json:"uri"` 34 | Method string `json:"method"` 35 | Complete bool `json:"complete"` 36 | } 37 | 38 | func (rq *Request) Success() bool { 39 | for _, item := range rq.Attempts { 40 | if item.Code == 0 { 41 | return true 42 | } 43 | } 44 | return false 45 | } 46 | -------------------------------------------------------------------------------- /services/meta/micrometa/request_meta_storage.go: -------------------------------------------------------------------------------- 1 | package micrometa 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | "time" 10 | 11 | "github.com/dgraph-io/badger" 12 | 13 | "nano-run/internal" 14 | "nano-run/services/meta" 15 | ) 16 | 17 | func WrapMetaStorage(db *badger.DB) *MicroMeta { 18 | return &MicroMeta{db: db, wrapped: true} 19 | } 20 | 21 | func NewMetaStorage(location string) (*MicroMeta, error) { 22 | name := filepath.Base(location) 23 | db, err := badger.Open(badger.DefaultOptions(location). 24 | WithLogger(internal.NanoLogger(log.New(os.Stderr, "["+name+"] ", log.LstdFlags)))) 25 | if err != nil { 26 | return nil, err 27 | } 28 | return &MicroMeta{ 29 | db: db, 30 | wrapped: false, 31 | }, nil 32 | } 33 | 34 | type MicroMeta struct { 35 | db *badger.DB 36 | wrapped bool 37 | } 38 | 39 | func (rms *MicroMeta) Get(requestID string) (*meta.Request, error) { 40 | var ans meta.Request 41 | return &ans, rms.db.View(func(txn *badger.Txn) error { 42 | item, err := txn.Get([]byte(requestID)) 43 | if err != nil { 44 | return err 45 | } 46 | return item.Value(func(val []byte) error { 47 | return json.Unmarshal(val, &ans) 48 | }) 49 | }) 50 | } 51 | 52 | func (rms *MicroMeta) CreateRequest(requestID string, headers http.Header, uri string, method string) error { 53 | var record = meta.Request{ 54 | CreatedAt: time.Now(), 55 | Attempts: make([]meta.Attempt, 0), 56 | Complete: false, 57 | Headers: headers, 58 | URI: uri, 59 | Method: method, 60 | } 61 | data, err := json.Marshal(record) 62 | if err != nil { 63 | return err 64 | } 65 | return rms.db.Update(func(txn *badger.Txn) error { 66 | return txn.Set([]byte(requestID), data) 67 | }) 68 | } 69 | 70 | func (rms *MicroMeta) AddAttempt(requestID, attemptID string, header meta.AttemptHeader) (*meta.Request, error) { 71 | var ans meta.Request 72 | err := rms.updateRequest(requestID, func(record *meta.Request) error { 73 | record.Attempts = append(record.Attempts, meta.Attempt{ 74 | ID: attemptID, 75 | CreatedAt: time.Now(), 76 | AttemptHeader: header, 77 | }) 78 | ans = *record 79 | return nil 80 | }) 81 | return &ans, err 82 | } 83 | 84 | func (rms *MicroMeta) Complete(requestID string) error { 85 | return rms.updateRequest(requestID, func(record *meta.Request) error { 86 | record.Complete = true 87 | record.CompleteAt = time.Now() 88 | return nil 89 | }) 90 | } 91 | 92 | func (rms *MicroMeta) Close() error { 93 | if rms.wrapped { 94 | return nil 95 | } 96 | return rms.db.Close() 97 | } 98 | 99 | func (rms *MicroMeta) Iterate(handler func(id string, record meta.Request) error) error { 100 | return rms.db.View(func(txn *badger.Txn) error { 101 | cfg := badger.DefaultIteratorOptions 102 | cfg.Reverse = true 103 | iter := txn.NewIterator(cfg) 104 | iter.Rewind() 105 | defer iter.Close() 106 | for iter.Valid() { 107 | id := string(iter.Item().Key()) 108 | var rec meta.Request 109 | err := iter.Item().Value(func(val []byte) error { 110 | return json.Unmarshal(val, &rec) 111 | }) 112 | if err != nil { 113 | return err 114 | } 115 | err = handler(id, rec) 116 | if err != nil { 117 | return err 118 | } 119 | iter.Next() 120 | } 121 | return nil 122 | }) 123 | } 124 | 125 | func (rms *MicroMeta) updateRequest(requestID string, tx func(record *meta.Request) error) error { 126 | return rms.db.Update(func(txn *badger.Txn) error { 127 | data, err := txn.Get([]byte(requestID)) 128 | if err != nil { 129 | return err 130 | } 131 | var record meta.Request 132 | err = data.Value(func(val []byte) error { 133 | return json.Unmarshal(val, &record) 134 | }) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | err = tx(&record) 140 | if err != nil { 141 | return err 142 | } 143 | 144 | value, err := json.Marshal(record) 145 | if err != nil { 146 | return err 147 | } 148 | return txn.Set([]byte(requestID), value) 149 | }) 150 | } 151 | -------------------------------------------------------------------------------- /services/queue/interface.go: -------------------------------------------------------------------------------- 1 | package queue 2 | 3 | import "context" 4 | 5 | type Queue interface { 6 | Push(payload []byte) error 7 | Get(ctx context.Context) ([]byte, error) 8 | } 9 | -------------------------------------------------------------------------------- /services/queue/microqueue/micro_queue.go: -------------------------------------------------------------------------------- 1 | package microqueue 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "errors" 7 | "log" 8 | "os" 9 | "path/filepath" 10 | "sync/atomic" 11 | 12 | "github.com/dgraph-io/badger" 13 | 14 | "nano-run/internal" 15 | ) 16 | 17 | var ErrEmptyQueue = errors.New("empty queue") 18 | 19 | func WrapMicroQueue(db *badger.DB) (*MicroQueue, error) { 20 | mc := &MicroQueue{db: db, wrapped: true, notify: make(chan struct{}, 1), close: make(chan struct{})} 21 | return mc, mc.db.DropAll() 22 | } 23 | 24 | func NewMicroQueue(location string) (*MicroQueue, error) { 25 | name := filepath.Base(location) 26 | db, err := badger.Open(badger.DefaultOptions(location).WithTruncate(true).WithLogger(internal.NanoLogger(log.New(os.Stderr, "["+name+"] ", log.LstdFlags)))) 27 | if err != nil { 28 | return nil, err 29 | } 30 | return &MicroQueue{db: db, notify: make(chan struct{}, 1), close: make(chan struct{})}, nil 31 | } 32 | 33 | // Always fresh queue with offloading to fs if no readers. 34 | // Optimized for multiple readers and multiple writers with number of items limited by FS 35 | // and each value should fit to RAM. 36 | type MicroQueue struct { 37 | db *badger.DB 38 | sequence uint64 39 | wrapped bool 40 | notify chan struct{} 41 | close chan struct{} 42 | } 43 | 44 | func (mq *MicroQueue) Push(payload []byte) error { 45 | id := atomic.AddUint64(&mq.sequence, 1) 46 | var key [8]byte 47 | binary.BigEndian.PutUint64(key[:], id) 48 | err := mq.db.Update(func(txn *badger.Txn) error { 49 | return txn.Set(key[:], payload) 50 | }) 51 | if err != nil { 52 | return err 53 | } 54 | mq.sendNotify() 55 | return nil 56 | } 57 | 58 | func (mq *MicroQueue) pop() ([]byte, error) { 59 | var ans []byte 60 | err := mq.db.Update(func(txn *badger.Txn) error { 61 | it := txn.NewIterator(badger.IteratorOptions{ 62 | PrefetchValues: true, 63 | PrefetchSize: 1, 64 | Reverse: false, 65 | AllVersions: false, 66 | }) 67 | defer it.Close() 68 | it.Rewind() 69 | 70 | if !it.Valid() { 71 | return ErrEmptyQueue 72 | } 73 | v, err := it.Item().ValueCopy(ans) 74 | if err != nil { 75 | return err 76 | } 77 | ans = v 78 | 79 | return txn.Delete(it.Item().Key()) 80 | }) 81 | if err != nil { 82 | return nil, err 83 | } 84 | mq.sendNotify() 85 | return ans, nil 86 | } 87 | 88 | // Get blocking for new item in a queue. 89 | func (mq *MicroQueue) Get(ctx context.Context) ([]byte, error) { 90 | mq.sendNotify() 91 | for { 92 | select { 93 | case <-ctx.Done(): 94 | return nil, ctx.Err() 95 | case <-mq.close: 96 | return nil, errors.New("queue closed") 97 | case <-mq.notify: 98 | v, err := mq.pop() 99 | if err == nil { 100 | return v, nil 101 | } 102 | if errors.Is(err, ErrEmptyQueue) { 103 | continue 104 | } 105 | return nil, err 106 | } 107 | } 108 | } 109 | 110 | func (mq *MicroQueue) Close() error { 111 | close(mq.close) 112 | if mq.wrapped { 113 | return nil 114 | } 115 | return mq.db.Close() 116 | } 117 | 118 | func (mq *MicroQueue) sendNotify() { 119 | select { 120 | case mq.notify <- struct{}{}: 121 | default: 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /templates/cron-list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 | 28 |
29 |
30 |
31 |
32 |
33 |
All cron
34 |
35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | {{range .Entries}} 45 | 46 | 49 | 52 | 55 | 56 | {{end}} 57 | 58 |
UnitNameSpec
47 | {{.Entry.Unit.Name}} 48 | 50 | {{.Entry.Spec.Label .Entry.Name}} 51 | 53 | {{.Entry.Spec.Spec}} 54 |
59 |
60 |
61 |
62 |
63 |
64 |
65 | 66 | 67 | -------------------------------------------------------------------------------- /templates/login.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 |
10 |
11 |
12 |
13 |
Login to the system
14 |
15 | {{with .Auth.OAuth2}} 16 | Login by {{.Title}} 17 | {{end}} 18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 | 26 | 27 | -------------------------------------------------------------------------------- /templates/unit-cron-info.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |
8 | 27 | 34 |
35 |
36 |
37 |
38 |
Scheduled job {{.Label}}
39 |
Configuration
40 |
41 |
42 |
Spec
43 |
44 | {{.Cron.Spec.Spec}} 45 |
46 | {{if .Cron.Spec.Content}} 47 |
Content
48 | 
{{.Cron.Spec.Content}}
49 | {{else if .Cron.Spec.ContentFile}} 50 |
Content file
51 |
{{.Cron.Spec.ContentFile}}
52 | {{end}} 53 |
54 |
55 | {{if .Cron.Spec.Headers}} 56 |
Headers
57 |
58 | {{with .Cron.Spec.Headers}} 59 |
60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | {{range $k,$v := .}} 69 | 70 | 73 | 74 | 75 | {{end}} 76 | 77 |
NameValue
71 | 
{{$k}}
72 | 		{{$v}}
78 |
79 | {{else}} 80 | no custom variables defined 81 | {{end}} 82 |
83 | {{end}} 84 |
85 |
86 |
87 |
88 |
89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /templates/unit-info.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |
8 | 27 | 33 |
34 |
35 |
36 |
37 |
38 | {{.Unit.Name}} 39 |
40 |

41 | {{with .Unit}} 42 | {{if .Private}}🏠 43 | {{else if .Secured}}🛡️ 44 | {{end}} 45 | {{if gt (len .Cron) 0}} 46 | ⏲ 47 | {{end}} 48 | {{end}} 49 |

50 |
Configuration
51 |
52 |
53 | {{- if .Unit.Private}} 54 |
Private
55 |
yes - not exposed over API
56 | {{- else}} 57 |
API endpoint
58 |
59 | 60 | {{$.BaseURL}}/api/{{.Unit.Name}}/ 61 | 62 |
63 | {{- end}} 64 |
Mode
65 |
{{.Unit.Mode}}
66 |
Concurrency
67 |
{{.Unit.Workers}}
68 |
Attempts
69 |
{{.Unit.Attempts}}
70 |
Interval
71 |
{{.Unit.Interval}}
72 | {{if eq .Unit.Mode "bin"}} 73 |
Timeout
74 |
75 | {{with .Unit.Timeout}} 76 | {{.}} 77 | {{else}} 78 | ∞ 79 | {{end}} 80 |
81 |
Graceful timeout 82 |
83 |
84 | {{with .Unit.GracefulTimeout}} 85 | {{.}} 86 | {{else}} 87 | ∞ 88 | {{end}} 89 |
90 | {{end}} 91 |
Max request size
92 |
93 | {{with .Unit.MaxRequest}} 94 | {{.}} 95 | {{else}} 96 | ∞ 97 | {{end}} 98 |
99 |
Working directory
100 |
101 | {{with .Unit.WorkDir}} 102 | static 103 | {{.}} 104 | {{else}} 105 | dynamic 106 | {{end}} 107 |
108 | {{if or (eq .Unit.Mode "bin") (eq .Unit.Mode "cgi")}} 109 |
Command
110 | 
{{.Unit.Command}}
111 |
Shell
112 |
{{.Unit.Shell}}
113 | {{end}} 114 |
115 |
116 | {{with .Unit.Environment}} 117 |
Environment
118 |
119 |
120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | {{range $k,$v := .}} 129 | 130 | 133 | 134 | 135 | {{end}} 136 | 137 |
NameValue
131 | 
{{$k}}
132 | 		{{$v}}
138 |
139 |
140 |
141 | {{end}} 142 | {{if $.Unit.Secured}} 143 |
Authorization
144 |
145 | {{with .Unit.Authorization}} 146 |
147 | {{if .JWT.Enable}} 148 |
JWT
149 |
in 150 | {{.JWT.GetHeader}} 151 | header 152 |
153 | {{end}} 154 | {{if .QueryToken.Enable}} 155 |
Query token
156 |
157 |
158 | 159 | in 160 | {{.QueryToken.GetParam}} 161 | param 162 | 163 |
    164 | {{range .QueryToken.Tokens}} 165 |
  • 166 | {{$.BaseURL}}/api/{{$.Unit.Name}}/?{{$.Unit.Authorization.QueryToken.GetParam}}={{.}} 167 |
    • 168 | {{end}} 169 |
170 |
171 | 172 |
173 | {{end}} 174 | {{if .HeaderToken.Enable}} 175 |
Header token
176 |
in 177 | {{.HeaderToken.GetHeader}} 178 | header 179 |
180 | {{end}} 181 | {{if .Basic.Enable}} 182 |
Basic
183 |
{{.Basic.Logins | join ", "}}
184 | {{end}} 185 |
186 | {{end}} 187 |
188 | {{end}} 189 |
190 |
191 |
192 |
193 | {{with .CronEntries}} 194 |
195 |
196 |
197 |
198 |
199 |
200 |
Schedules
201 |
202 |
203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | {{range $index, $entry := .}} 212 | 213 | 216 | 219 | 220 | {{end}} 221 | 222 |
NameSpec
214 | {{$entry.Spec.Label (print "#" $index)}} 215 | 217 | {{$entry.Spec.Spec}} 218 |
223 |
224 |
225 | 226 |
227 |
228 |
229 |
230 |
231 | {{end}} 232 |
233 |
234 |
235 |
236 |
237 |
238 |
Try
239 |
240 | 241 |
242 |
243 | 244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
252 |
253 |
254 |
Last 50 requests
255 |
256 |
257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | {{range $.History 50}} 268 | 269 | 272 | 273 | 280 | 281 | 282 | {{end}} 283 | 284 |
IDTimeCompleteAttempts
270 | {{.ID}} 271 | {{.Meta.CreatedAt.Format "02 Jan 06 15:04:05.000 MST"}} 274 | {{if .Meta.Complete}} 275 | {{.Meta.CompleteAt.Format "02 Jan 06 15:04:05.000 MST"}} 276 | {{else}} 277 | in progress 278 | {{end}} 279 | {{len .Meta.Attempts}}
285 |
286 |
287 |
288 |
289 |
290 |
291 |
292 | 293 | 294 | 295 | -------------------------------------------------------------------------------- /templates/unit-request-attempt-info.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |
8 | 27 | 36 |
37 |
38 |
39 |
40 |
Attempt
41 |
Meta info
42 |
43 |
44 |
ID
45 |
{{.AttemptID}}
46 |
Started at
47 |
48 | {{.Attempt.StartedAt.Format "02 Jan 06 15:04:05.000 MST"}} 49 |
50 |
Finished at
51 |
52 | {{.Attempt.CreatedAt.Format "02 Jan 06 15:04:05.000 MST"}} 53 |
after {{.Attempt.CreatedAt.Sub .Attempt.StartedAt}}
54 |
55 |
Code
56 |
{{.Attempt.Code}}
57 |
58 |
59 |
Headers
60 |
61 |
62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | {{range $k, $v := .Attempt.Headers}} 71 | 72 | 73 | 74 | 75 | {{end}} 76 | 77 |
NameValue
{{$k}}{{$v | join ", "}}
78 |
79 |
80 | 82 | open result in a new 83 | tab 84 |
85 |
86 |
87 |
88 |
89 | 108 |
109 | 137 | 138 | 139 | -------------------------------------------------------------------------------- /templates/unit-request-info.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |
8 | 27 | 34 |
35 |
36 |
37 |
38 |
Request
39 |
Meta info
40 |
41 |
42 |
ID
43 |
{{.RequestID}}
44 |
Created at
45 |
{{.Request.CreatedAt.Format "02 Jan 06 15:04:05.000 MST"}}
46 |
Complete at
47 |
48 | {{if .Request.Complete}} 49 | {{.Request.CompleteAt.Format "02 Jan 06 15:04:05.000 MST"}} 50 |
after {{.Request.CompleteAt.Sub .Request.CreatedAt}}
51 | {{else}} 52 | in progress 53 | {{end}} 54 |
55 |
URI
56 |
{{.Request.URI}}
57 |
Method
58 |
{{.Request.Method}}
59 |
60 |
61 |
Headers
62 |
63 |
64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | {{range $k,$v := .Request.Headers}} 73 | 74 | 75 | 78 | 79 | {{end}} 80 | 81 |
NameValue
{{$k}} 76 | {{$v | join ", "}} 77 |
82 |
83 |
84 |
85 | 86 |
87 | 89 | open payload in a new tab 90 |
91 |
92 |
93 |
94 | 117 |
118 |
119 |
120 |
121 |
122 |
Attempts
123 |
124 |
125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | {{range .Request.Attempts}} 135 | 136 | 137 | 138 | 139 | 140 | {{end}} 141 | 142 |
IDTimeCode
{{.ID}}{{.CreatedAt.Format "02 Jan 06 15:04:05.000 MST"}}{{.Code}}
143 |
144 |
145 |
146 | 147 |
148 |
149 |
150 |
151 | 179 | 180 | 181 | -------------------------------------------------------------------------------- /templates/units-list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |
9 | 28 |
29 |
30 |
31 |
32 |
33 |
All units
34 |
35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | {{range .Units}} 51 | 52 | 62 | 65 | 68 | 71 | 74 | 77 | 84 | 91 | 101 | 102 | {{end}} 103 | 104 |
NameModeConcurrencyAttemptsIntervalTimeoutMax request sizeWorking directory
53 | {{with .}} 54 | {{if .Private}}🏠 55 | {{else if .Secured}}🛡️ 56 | {{end}} 57 | {{if gt (len .Cron) 0}} 58 | ⏲ 59 | {{end}} 60 | {{end}} 61 | 63 | {{.Name}} 64 | 66 | {{.Mode}} 67 | 69 | {{.Workers}} 70 | 72 | {{.Attempts}} 73 | 75 | {{.Interval}} 76 | 78 | {{with .Timeout}} 79 | {{.}} 80 | {{else}} 81 | ∞ 82 | {{end}} 83 | 85 | {{with .MaxRequest}} 86 | {{.}} 87 | {{else}} 88 | ∞ 89 | {{end}} 90 | 92 | {{with .WorkDir}} 93 |
94 | static 95 |

{{.}}

96 |
97 | {{else}} 98 | dynamic 99 | {{end}} 100 |
105 |
106 |
107 |
108 |
109 |
110 |
111 | 112 | 113 | -------------------------------------------------------------------------------- /worker/doc.go: -------------------------------------------------------------------------------- 1 | // Processing unit 2 | // 3 | // Worker manages requests, queuing and re-queuing. 4 | package worker 5 | -------------------------------------------------------------------------------- /worker/request_io.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "io" 5 | "net/http" 6 | 7 | "nano-run/services/meta" 8 | ) 9 | 10 | func openResponse(writer io.Writer) *responseStream { 11 | return &responseStream{stream: writer, meta: meta.AttemptHeader{Headers: make(http.Header)}} 12 | } 13 | 14 | type responseStream struct { 15 | meta meta.AttemptHeader 16 | statusSent bool 17 | stream io.Writer 18 | } 19 | 20 | func (mo *responseStream) Header() http.Header { 21 | return mo.meta.Headers 22 | } 23 | 24 | func (mo *responseStream) Write(bytes []byte) (int, error) { 25 | if !mo.statusSent { 26 | mo.WriteHeader(http.StatusOK) 27 | } 28 | return mo.stream.Write(bytes) 29 | } 30 | 31 | func (mo *responseStream) WriteHeader(statusCode int) { 32 | if mo.statusSent { 33 | return 34 | } 35 | mo.statusSent = true 36 | mo.meta.Code = statusCode 37 | } 38 | 39 | func (mo *responseStream) Status(status int) { 40 | mo.meta.Code = status 41 | } 42 | -------------------------------------------------------------------------------- /worker/tracker.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import "sync/atomic" 4 | 5 | func newTracker(id string) *Tracker { 6 | return &Tracker{ 7 | id: id, 8 | done: make(chan struct{}), 9 | } 10 | } 11 | 12 | type Tracker struct { 13 | id string 14 | done chan struct{} 15 | success bool 16 | attemptID string 17 | finished int32 18 | } 19 | 20 | func (t *Tracker) ID() string { 21 | return t.id 22 | } 23 | 24 | func (t *Tracker) Done() <-chan struct{} { 25 | return t.done 26 | } 27 | 28 | func (t *Tracker) Success() bool { 29 | return t.success 30 | } 31 | 32 | func (t *Tracker) Attempt() string { 33 | return t.attemptID 34 | } 35 | 36 | func (t *Tracker) close() { 37 | if atomic.CompareAndSwapInt32(&t.finished, 0, 1) { 38 | close(t.done) 39 | } 40 | } 41 | 42 | func (t *Tracker) ok(attemptID string) { 43 | t.attemptID = attemptID 44 | t.success = true 45 | t.close() 46 | } 47 | 48 | func (t *Tracker) failed() { 49 | t.success = false 50 | t.close() 51 | } 52 | -------------------------------------------------------------------------------- /worker/worker.go: -------------------------------------------------------------------------------- 1 | package worker 2 | 3 | import ( 4 | "context" 5 | "encoding/binary" 6 | "encoding/hex" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "log" 12 | "net/http" 13 | "os" 14 | "path/filepath" 15 | "regexp" 16 | "runtime" 17 | "strconv" 18 | "strings" 19 | "sync" 20 | "sync/atomic" 21 | "time" 22 | 23 | "nano-run/services/blob" 24 | "nano-run/services/blob/fsblob" 25 | "nano-run/services/meta" 26 | "nano-run/services/meta/micrometa" 27 | "nano-run/services/queue" 28 | "nano-run/services/queue/microqueue" 29 | ) 30 | 31 | type ( 32 | CompleteHandler func(ctx context.Context, requestID string, info *meta.Request) 33 | ProcessHandler func(ctx context.Context, requestID, attemptID string, info *meta.Request) 34 | ) 35 | 36 | const ( 37 | defaultAttempts = 3 38 | defaultInterval = 3 * time.Second 39 | minimalFailedCode = 500 40 | nsRequest byte = 0x00 41 | nsAttempt byte = 0x01 42 | ) 43 | 44 | func Default(location string) (*Worker, error) { 45 | path := filepath.Join(location, "blobs") 46 | err := os.MkdirAll(path, 0755) 47 | if err != nil { 48 | return nil, err 49 | } 50 | valid, err := regexp.Compile("^[a-zA-Z0-9-]+$") 51 | if err != nil { 52 | return nil, err 53 | } 54 | storage := fsblob.NewCheck(path, func(id string) bool { 55 | return valid.MatchString(id) 56 | }) 57 | 58 | taskQueue, err := microqueue.NewMicroQueue(filepath.Join(location, "queue")) 59 | if err != nil { 60 | return nil, err 61 | } 62 | requeue, err := microqueue.NewMicroQueue(filepath.Join(location, "requeue")) 63 | if err != nil { 64 | return nil, err 65 | } 66 | metaStorage, err := micrometa.NewMetaStorage(filepath.Join(location, "meta")) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | cleanup := func() { 72 | _ = requeue.Close() 73 | _ = taskQueue.Close() 74 | _ = metaStorage.Close() 75 | } 76 | 77 | wrk, err := New(taskQueue, requeue, storage, metaStorage) 78 | if err != nil { 79 | cleanup() 80 | return nil, err 81 | } 82 | wrk.cleanup = cleanup 83 | return wrk, nil 84 | } 85 | 86 | func New(tasks, requeue queue.Queue, blobs blob.Blob, meta meta.Meta) (*Worker, error) { 87 | wrk := &Worker{ 88 | queue: tasks, 89 | requeue: requeue, 90 | blob: blobs, 91 | meta: meta, 92 | reloadMeta: make(chan struct{}, 1), 93 | maxAttempts: defaultAttempts, 94 | interval: defaultInterval, 95 | concurrency: runtime.NumCPU(), 96 | handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { 97 | writer.WriteHeader(http.StatusNoContent) 98 | }), 99 | } 100 | err := wrk.init() 101 | if err != nil { 102 | return nil, err 103 | } 104 | return wrk, nil 105 | } 106 | 107 | type Worker struct { 108 | queue queue.Queue 109 | requeue queue.Queue 110 | blob blob.Blob 111 | meta meta.Meta 112 | handler http.Handler 113 | cleanup func() 114 | 115 | onDead CompleteHandler 116 | onSuccess CompleteHandler 117 | onProcess ProcessHandler 118 | maxAttempts int 119 | concurrency int 120 | reloadMeta chan struct{} 121 | interval time.Duration 122 | sequence uint64 123 | trackers sync.Map // id -> *Tracker 124 | } 125 | 126 | func (mgr *Worker) init() error { 127 | return mgr.meta.Iterate(func(id string, record meta.Request) error { 128 | if _, v, err := decodeID(id); err == nil && v > mgr.sequence { 129 | mgr.sequence = v 130 | } else if err != nil { 131 | log.Println("found broken id:", id, "-", err) 132 | } 133 | if !record.Complete { 134 | log.Println("found incomplete job", id) 135 | return mgr.queue.Push([]byte(id)) 136 | } 137 | return nil 138 | }) 139 | } 140 | 141 | // Cleanup internal resource. 142 | func (mgr *Worker) Close() { 143 | if fn := mgr.cleanup; fn != nil { 144 | fn() 145 | } 146 | } 147 | 148 | // Enqueue request to storage, save meta-info to meta storage and push id into processing queue. Generated ID 149 | // always unique and returns only in case of successful enqueue. 150 | func (mgr *Worker) Enqueue(req *http.Request) (string, error) { 151 | id, err := mgr.saveRequest(req) 152 | if err != nil { 153 | return "", err 154 | } 155 | log.Println("new request saved:", id) 156 | err = mgr.queue.Push([]byte(id)) 157 | return id, err 158 | } 159 | 160 | // Enqueue request to storage, save meta-info to meta storage and push id into processing queue. Generated ID 161 | // always unique and returns only in case of successful enqueue. Returns Tracker to understand job processing. 162 | // Tracking jobs is not free operation! Do not use it just because you can. 163 | func (mgr *Worker) EnqueueWithTracker(req *http.Request) (*Tracker, error) { 164 | id, err := mgr.saveRequest(req) 165 | if err != nil { 166 | return nil, err 167 | } 168 | track := newTracker(id) 169 | log.Println("new request saved:", id) 170 | mgr.trackers.Store(id, track) 171 | err = mgr.queue.Push([]byte(id)) 172 | if err != nil { 173 | track.close() 174 | mgr.trackers.Delete(id) 175 | return nil, err 176 | } 177 | return track, nil 178 | } 179 | 180 | // Complete request manually. 181 | func (mgr *Worker) Complete(requestID string) error { 182 | err := mgr.meta.Complete(requestID) 183 | if err != nil { 184 | return err 185 | } 186 | select { 187 | case mgr.reloadMeta <- struct{}{}: 188 | default: 189 | } 190 | return nil 191 | } 192 | 193 | func (mgr *Worker) OnSuccess(handler CompleteHandler) *Worker { 194 | mgr.onSuccess = handler 195 | return mgr 196 | } 197 | 198 | func (mgr *Worker) OnDead(handler CompleteHandler) *Worker { 199 | mgr.onDead = handler 200 | return mgr 201 | } 202 | 203 | func (mgr *Worker) OnProcess(handler ProcessHandler) *Worker { 204 | mgr.onProcess = handler 205 | return mgr 206 | } 207 | 208 | func (mgr *Worker) Handler(handler http.Handler) *Worker { 209 | mgr.handler = handler 210 | return mgr 211 | } 212 | 213 | func (mgr *Worker) HandlerFunc(fn http.HandlerFunc) *Worker { 214 | mgr.handler = fn 215 | return mgr 216 | } 217 | 218 | // Attempts number of 500x requests. 219 | func (mgr *Worker) Attempts(max int) *Worker { 220 | mgr.maxAttempts = max 221 | return mgr 222 | } 223 | 224 | // Interval between attempts. 225 | func (mgr *Worker) Interval(duration time.Duration) *Worker { 226 | mgr.interval = duration 227 | return mgr 228 | } 229 | 230 | // Concurrency limit (number of parallel tasks). Does not affect already running worker. 231 | // 0 means num CPU. 232 | func (mgr *Worker) Concurrency(num int) *Worker { 233 | mgr.concurrency = num 234 | if num == 0 { 235 | mgr.concurrency = runtime.NumCPU() 236 | } 237 | return mgr 238 | } 239 | 240 | // Meta information about requests. 241 | func (mgr *Worker) Meta() meta.Meta { 242 | return mgr.meta 243 | } 244 | 245 | // Blobs storage (for large objects). 246 | func (mgr *Worker) Blobs() blob.Blob { 247 | return mgr.blob 248 | } 249 | 250 | func (mgr *Worker) Run(global context.Context) error { 251 | if mgr.interval < 0 { 252 | return fmt.Errorf("negative interval") 253 | } 254 | if mgr.maxAttempts < 0 { 255 | return fmt.Errorf("negative attempts") 256 | } 257 | if mgr.handler == nil { 258 | return fmt.Errorf("nil handler") 259 | } 260 | if mgr.concurrency <= 0 { 261 | return fmt.Errorf("invalid concurrency number") 262 | } 263 | ctx, cancel := context.WithCancel(global) 264 | defer cancel() 265 | var wg sync.WaitGroup 266 | for i := 0; i < mgr.concurrency; i++ { 267 | wg.Add(1) 268 | go func(i int) { 269 | defer wg.Done() 270 | defer cancel() 271 | err := mgr.runQueue(ctx) 272 | if err != nil { 273 | log.Println("worker", i, "stopped due to error:", err) 274 | } else { 275 | log.Println("worker", i, "stopped") 276 | } 277 | }(i) 278 | } 279 | wg.Add(1) 280 | go func() { 281 | defer wg.Done() 282 | defer cancel() 283 | err := mgr.runReQueue(ctx) 284 | if err != nil { 285 | log.Println("re-queue process stopped due to error:", err) 286 | } else { 287 | log.Println("re-queue process stopped") 288 | } 289 | }() 290 | wg.Wait() 291 | return ctx.Err() 292 | } 293 | 294 | // Retry processing. 295 | func (mgr *Worker) Retry(ctx context.Context, requestID string) (string, error) { 296 | info, err := mgr.meta.Get(requestID) 297 | if err != nil { 298 | return "", err 299 | } 300 | req, err := mgr.restoreRequest(ctx, requestID, info) 301 | if err != nil { 302 | return "", err 303 | } 304 | defer req.Body.Close() 305 | return mgr.Enqueue(req) 306 | } 307 | 308 | func (mgr *Worker) call(ctx context.Context, requestID string, info *meta.Request) error { 309 | // caller should ensure that request id is valid 310 | req, err := mgr.restoreRequest(ctx, requestID, info) 311 | if err != nil { 312 | return err 313 | } 314 | defer req.Body.Close() 315 | attemptID := encodeID(nsAttempt, uint64(len(info.Attempts))+1) 316 | 317 | req.Header.Set("X-Correlation-Id", requestID) 318 | req.Header.Set("X-Attempt-Id", attemptID) 319 | req.Header.Set("X-Attempt", strconv.Itoa(len(info.Attempts)+1)) 320 | 321 | var header meta.AttemptHeader 322 | 323 | err = mgr.blob.Push(attemptID, func(out io.Writer) error { 324 | res := openResponse(out) 325 | started := time.Now() 326 | mgr.handler.ServeHTTP(res, req) 327 | header = res.meta 328 | header.StartedAt = started 329 | return nil 330 | }) 331 | if err != nil { 332 | return err 333 | } 334 | 335 | info, err = mgr.meta.AddAttempt(requestID, attemptID, header) 336 | if err != nil { 337 | return err 338 | } 339 | 340 | mgr.requestProcessed(ctx, requestID, attemptID, info) 341 | if header.Code >= minimalFailedCode { 342 | return fmt.Errorf("500 code returned: %d", header.Code) 343 | } 344 | return nil 345 | } 346 | 347 | func (mgr *Worker) runQueue(ctx context.Context) error { 348 | for { 349 | err := mgr.processQueueItem(ctx) 350 | if err != nil { 351 | return err 352 | } 353 | select { 354 | case <-ctx.Done(): 355 | return ctx.Err() 356 | default: 357 | } 358 | } 359 | } 360 | 361 | func (mgr *Worker) runReQueue(ctx context.Context) error { 362 | for { 363 | err := mgr.processReQueueItem(ctx) 364 | if err != nil { 365 | return err 366 | } 367 | select { 368 | case <-ctx.Done(): 369 | return ctx.Err() 370 | default: 371 | } 372 | } 373 | } 374 | 375 | func (mgr *Worker) processQueueItem(ctx context.Context) error { 376 | bid, err := mgr.queue.Get(ctx) 377 | if err != nil { 378 | return err 379 | } 380 | id := string(bid) 381 | log.Println("processing request", id) 382 | info, err := mgr.meta.Get(id) 383 | if err != nil { 384 | return fmt.Errorf("get request %s meta info: %w", id, err) 385 | } 386 | if info.Complete { 387 | log.Printf("request %s already complete", id) 388 | return nil 389 | } 390 | err = mgr.call(ctx, id, info) 391 | if err == nil { 392 | mgr.requestSuccess(ctx, id, info) 393 | return nil 394 | } 395 | return mgr.requeueItem(ctx, id, info) 396 | } 397 | 398 | func (mgr *Worker) processReQueueItem(ctx context.Context) error { 399 | var item requeueItem 400 | 401 | data, err := mgr.requeue.Get(ctx) 402 | if err != nil { 403 | return err 404 | } 405 | err = json.Unmarshal(data, &item) 406 | 407 | if err != nil { 408 | return err 409 | } 410 | 411 | d := time.Since(item.At) 412 | if d < mgr.interval { 413 | var ok = false 414 | for !ok { 415 | info, err := mgr.meta.Get(item.ID) 416 | if err != nil { 417 | return fmt.Errorf("re-queue: get meta %s: %w", item.ID, err) 418 | } 419 | if info.Complete { 420 | log.Printf("re-queue: %s already complete", item.ID) 421 | return nil 422 | } 423 | select { 424 | case <-time.After(mgr.interval - d): 425 | ok = true 426 | case <-mgr.reloadMeta: 427 | case <-ctx.Done(): 428 | return ctx.Err() 429 | } 430 | } 431 | } 432 | return mgr.queue.Push([]byte(item.ID)) 433 | } 434 | 435 | func (mgr *Worker) requeueItem(ctx context.Context, id string, info *meta.Request) error { 436 | if len(info.Attempts) >= mgr.maxAttempts { 437 | mgr.requestDead(ctx, id, info) 438 | log.Println("maximum attempts reached for request", id) 439 | return nil 440 | } 441 | data, err := json.Marshal(requeueItem{ 442 | At: time.Now(), 443 | ID: id, 444 | }) 445 | if err != nil { 446 | return err 447 | } 448 | return mgr.requeue.Push(data) 449 | } 450 | 451 | func (mgr *Worker) saveRequest(req *http.Request) (string, error) { 452 | id := encodeID(nsRequest, atomic.AddUint64(&mgr.sequence, 1)) 453 | err := mgr.blob.Push(id, func(out io.Writer) error { 454 | _, err := io.Copy(out, req.Body) 455 | return err 456 | }) 457 | if err != nil { 458 | return "", err 459 | } 460 | return id, mgr.meta.CreateRequest(id, req.Header, req.URL.RequestURI(), req.Method) 461 | } 462 | 463 | func (mgr *Worker) requestDead(ctx context.Context, id string, info *meta.Request) { 464 | err := mgr.meta.Complete(id) 465 | if err != nil { 466 | log.Println("failed complete (dead) request:", err) 467 | } 468 | if handler := mgr.onDead; handler != nil { 469 | handler(ctx, id, info) 470 | } 471 | log.Println("request", id, "completely failed") 472 | mgr.completeTrack(id, "", true) 473 | } 474 | 475 | func (mgr *Worker) requestSuccess(ctx context.Context, id string, info *meta.Request) { 476 | err := mgr.meta.Complete(id) 477 | if err != nil { 478 | log.Println("failed complete (success) request:", err) 479 | } 480 | if handler := mgr.onSuccess; handler != nil { 481 | handler(ctx, id, info) 482 | } 483 | log.Println("request", id, "complete successfully") 484 | } 485 | 486 | func (mgr *Worker) requestProcessed(ctx context.Context, id string, attemptID string, info *meta.Request) { 487 | if handler := mgr.onProcess; handler != nil { 488 | handler(ctx, id, attemptID, info) 489 | } 490 | log.Println("request", id, "processed with attempt", attemptID) 491 | mgr.completeTrack(id, attemptID, info.Success()) 492 | } 493 | 494 | func (mgr *Worker) restoreRequest(ctx context.Context, requestID string, info *meta.Request) (*http.Request, error) { 495 | f, err := mgr.blob.Get(requestID) 496 | if err != nil { 497 | return nil, err 498 | } 499 | req, err := http.NewRequestWithContext(ctx, info.Method, info.URI, f) 500 | if err != nil { 501 | _ = f.Close() 502 | return nil, err 503 | } 504 | for k, v := range info.Headers { 505 | req.Header[k] = v 506 | } 507 | return req, nil 508 | } 509 | 510 | func (mgr *Worker) completeTrack(id string, attemptID string, failed bool) { 511 | track, ok := mgr.trackers.LoadAndDelete(id) 512 | if !ok { 513 | return 514 | } 515 | tracker := track.(*Tracker) 516 | if failed { 517 | tracker.failed() 518 | } else { 519 | tracker.ok(attemptID) 520 | } 521 | } 522 | 523 | func encodeID(nsID byte, id uint64) string { 524 | var data [9]byte 525 | data[0] = nsID 526 | binary.BigEndian.PutUint64(data[1:], id) 527 | return strings.ToUpper(hex.EncodeToString(data[:])) 528 | } 529 | 530 | func decodeID(val string) (byte, uint64, error) { 531 | const idLen = 1 + 8 532 | hx, err := hex.DecodeString(val) 533 | if err != nil { 534 | return 0, 0, err 535 | } 536 | if len(hx) != idLen { 537 | return 0, 0, errors.New("too short") 538 | } 539 | n := binary.BigEndian.Uint64(hx[1:]) 540 | return hx[0], n, nil 541 | } 542 | 543 | type requeueItem struct { 544 | At time.Time 545 | ID string 546 | } 547 | --------------------------------------------------------------------------------