├── .envrc ├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ ├── automerge.yml │ └── build.yml ├── .gitignore ├── .golangci.yml ├── .pre-commit-config.yaml ├── .secrets.baseline ├── LICENSE ├── Makefile ├── README.md ├── atom.go ├── atomizer.go ├── atomizer_exported.go ├── atomizer_mock_test.go ├── atomizer_test.go ├── conductor.go ├── docs ├── definitions.md ├── design-methodologies.md └── media │ ├── design.jpg │ └── design_small.jpg ├── electron.go ├── electron_test.go ├── error.go ├── error_test.go ├── event.go ├── event_test.go ├── flake.lock ├── flake.nix ├── go.mod ├── go.sum ├── helpers.go ├── instance.go ├── instance_test.go ├── properties.go ├── properties_test.go ├── register.go ├── register_mock_test.go └── register_test.go /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @spyderorg/maintainers 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: gomod 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | labels: 8 | - "automerge" 9 | - package-ecosystem: "github-actions" 10 | directory: "/" 11 | schedule: 12 | interval: "daily" 13 | labels: 14 | - "automerge" 15 | - package-ecosystem: "docker" 16 | directory: "/" 17 | schedule: 18 | interval: "daily" 19 | labels: 20 | - "automerge" 21 | - package-ecosystem: "npm" 22 | directory: "/" 23 | schedule: 24 | interval: "daily" 25 | labels: 26 | - "automerge" 27 | - package-ecosystem: "pip" 28 | directory: "/" 29 | schedule: 30 | interval: "daily" 31 | labels: 32 | - "automerge" 33 | - package-ecosystem: "cargo" 34 | directory: "/" 35 | schedule: 36 | interval: "daily" 37 | labels: 38 | - "automerge" 39 | - package-ecosystem: "terraform" 40 | directory: "/" 41 | schedule: 42 | interval: "daily" 43 | labels: 44 | - "automerge" 45 | -------------------------------------------------------------------------------- /.github/workflows/automerge.yml: -------------------------------------------------------------------------------- 1 | name: automerge 2 | on: 3 | pull_request: 4 | types: 5 | - labeled 6 | - opened 7 | check_suite: 8 | types: 9 | - completed 10 | jobs: 11 | automerge: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - name: automerge 15 | uses: "pascalgn/automerge-action@v0.16.4" 16 | env: 17 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" 18 | MERGE_METHOD: squash 19 | MERGE_LABELS: automerge 20 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | env: 9 | GOPRIVATE: go.spyder.org 10 | 11 | jobs: 12 | build: 13 | name: "Build" 14 | runs-on: "ubuntu-latest" 15 | steps: 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | 19 | - uses: de-vri-es/setup-git-credentials@v2 20 | with: 21 | credentials: ${{secrets.GIT_CREDENTIALS}} 22 | 23 | - name: Install 1Password CLI 24 | uses: 1password/install-cli-action@v1 25 | 26 | - name: Install Nix 27 | uses: cachix/install-nix-action@v30 28 | with: 29 | extra_nix_config: | 30 | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} 31 | 32 | - uses: workflow/nix-shell-action@v3 33 | id: build 34 | with: 35 | flakes-from-devshell: true 36 | script: | 37 | make build-ci 38 | 39 | - name: Upload Test Coverage 40 | uses: actions/upload-artifact@v4 41 | with: 42 | name: coverage 43 | path: coverage.txt 44 | 45 | - name: Upload Fuzz Results 46 | uses: actions/upload-artifact@v4 47 | with: 48 | name: fuzz-results 49 | path: testdata/fuzz 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .direnv/ 2 | *cweb.log 3 | *.tmp 4 | .venv 5 | external/ 6 | 7 | # outputs 8 | output.txt 9 | dist/ 10 | 11 | # build folders 12 | bin/ 13 | ui/public/ 14 | ui/build/ 15 | ui/node_modules/ 16 | 17 | # Removing Vendor 18 | vendor/ 19 | 20 | .vscode/ 21 | 22 | # Binaries for programs and plugins 23 | *.exe 24 | *.exe~ 25 | *.dll 26 | *.so 27 | *.dylib 28 | 29 | # Test binary, build with `go test -c` 30 | *.test 31 | 32 | # Output of the go coverage tool, specifically when used with LiteIDE 33 | *.out 34 | coverage.html 35 | coverage 36 | coverage.txt 37 | 38 | # ignore keyfiles 39 | *.crt 40 | *.key 41 | *.pem 42 | 43 | # ignore mac files 44 | .DS_Store 45 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | timeout: 3m 3 | linters-settings: 4 | cyclop: 5 | max-complexity: 30 6 | package-average: 10.0 7 | 8 | errcheck: 9 | check-type-assertions: true 10 | 11 | exhaustive: 12 | check: 13 | - switch 14 | - map 15 | 16 | funlen: 17 | lines: 100 18 | statements: 50 19 | 20 | gocognit: 21 | min-complexity: 30 22 | 23 | gocritic: 24 | settings: 25 | captLocal: 26 | paramsOnly: false 27 | underef: 28 | skipRecvDeref: false 29 | 30 | mnd: 31 | ignored-functions: 32 | - os.Chmod 33 | - os.Mkdir 34 | - os.MkdirAll 35 | - os.OpenFile 36 | - os.WriteFile 37 | - prometheus.ExponentialBuckets 38 | - prometheus.ExponentialBucketsRange 39 | - prometheus.LinearBuckets 40 | 41 | gomodguard: 42 | blocked: 43 | modules: 44 | - github.com/golang/protobuf: 45 | recommendations: 46 | - google.golang.org/protobuf 47 | reason: "see https://developers.google.com/protocol-buffers/docs/reference/go/faq#modules" 48 | - github.com/satori/go.uuid: 49 | recommendations: 50 | - github.com/google/uuid 51 | reason: "satori's package is not maintained" 52 | - github.com/gofrs/uuid: 53 | recommendations: 54 | - github.com/google/uuid 55 | reason: "see recommendation from dev-infra team: https://confluence.gtforge.com/x/gQI6Aw" 56 | 57 | govet: 58 | enable-all: true 59 | disable: 60 | - fieldalignment 61 | - shadow 62 | nakedret: 63 | max-func-lines: 0 64 | 65 | nolintlint: 66 | allow-no-explanation: [ funlen, gocognit, lll ] 67 | require-explanation: true 68 | require-specific: true 69 | 70 | rowserrcheck: 71 | packages: 72 | - github.com/jmoiron/sqlx 73 | 74 | tenv: 75 | all: true 76 | 77 | tagliatelle: 78 | case: 79 | rules: 80 | json: snake 81 | yaml: camel 82 | xml: camel 83 | 84 | 85 | linters: 86 | disable-all: true 87 | enable: 88 | - errcheck 89 | - gosimple 90 | - govet 91 | - ineffassign 92 | - staticcheck 93 | - typecheck 94 | - unused 95 | - asasalint 96 | - asciicheck 97 | - bidichk 98 | - bodyclose 99 | - cyclop 100 | - dupl 101 | - durationcheck 102 | - errname 103 | - exhaustive 104 | - copyloopvar 105 | - forbidigo 106 | - funlen 107 | - gochecknoglobals 108 | - gocognit 109 | - goconst 110 | - gocritic 111 | - gocyclo 112 | - goimports 113 | - mnd 114 | - gomoddirectives 115 | - gomodguard 116 | - goprintffuncname 117 | - gosec 118 | - lll 119 | - loggercheck 120 | - makezero 121 | - nakedret 122 | - nestif 123 | - nilerr 124 | - nilnil 125 | - noctx 126 | - nolintlint 127 | - nosprintfhostport 128 | - predeclared 129 | - promlinter 130 | - reassign 131 | - revive 132 | - rowserrcheck 133 | - sqlclosecheck 134 | - stylecheck 135 | - tenv 136 | - testableexamples 137 | - tparallel 138 | - unconvert 139 | - unparam 140 | - usestdlibvars 141 | - wastedassign 142 | - whitespace 143 | issues: 144 | max-same-issues: 50 145 | 146 | exclude-dirs: 147 | - testdata 148 | 149 | exclude-rules: 150 | - source: "^//\\s*go:generate\\s" 151 | linters: [ lll ] 152 | - source: "(noinspection|TODO)" 153 | linters: [ godot ] 154 | - source: "//noinspection" 155 | linters: [ gocritic ] 156 | - source: "^\\s+if _, ok := err\\.\\([^.]+\\.InternalError\\); ok {" 157 | linters: [ errorlint ] 158 | - path: "_test\\.go" 159 | linters: 160 | - bodyclose 161 | - dupl 162 | - funlen 163 | - goconst 164 | - gosec 165 | - noctx 166 | - wrapcheck 167 | - mnd 168 | - copyloopref 169 | - gocyclo 170 | - errcheck 171 | - lll 172 | - gochecknoglobals 173 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: '^$' 2 | fail_fast: true 3 | repos: 4 | - repo: https://github.com/Yelp/detect-secrets 5 | rev: v1.5.0 6 | hooks: 7 | - id: detect-secrets 8 | name: Detect secrets 9 | language: python 10 | entry: detect-secrets-hook 11 | args: [ 12 | '--baseline', 13 | '.secrets.baseline', 14 | '--exclude-files', 15 | '(_test\.go$|/testdata/|gomod2nix.toml)', 16 | ] 17 | - repo: https://github.com/mrtazz/checkmake.git 18 | rev: 0.2.2 19 | hooks: 20 | - id: checkmake 21 | - repo: https://github.com/pre-commit/pre-commit-hooks 22 | rev: v5.0.0 23 | hooks: 24 | - id: check-json 25 | - id: check-merge-conflict 26 | - id: check-yaml 27 | - id: end-of-file-fixer 28 | - id: check-symlinks 29 | - repo: https://github.com/markdownlint/markdownlint 30 | rev: v0.12.0 31 | hooks: 32 | - id: markdownlint 33 | - repo: https://github.com/commitizen-tools/commitizen 34 | rev: v3.31.0 35 | hooks: 36 | - id: commitizen 37 | - id: commitizen-branch 38 | stages: [push] 39 | - repo: local 40 | hooks: 41 | - id: makefile 42 | name: Run Makefile Lint 43 | entry: make 44 | args: [pre-commit] 45 | language: system 46 | pass_filenames: false 47 | -------------------------------------------------------------------------------- /.secrets.baseline: -------------------------------------------------------------------------------- 1 | { 2 | "version": "1.1.2", 3 | "plugins_used": [ 4 | { 5 | "name": "ArtifactoryDetector" 6 | }, 7 | { 8 | "name": "AWSKeyDetector" 9 | }, 10 | { 11 | "name": "AzureStorageKeyDetector" 12 | }, 13 | { 14 | "name": "Base64HighEntropyString", 15 | "limit": 4.5 16 | }, 17 | { 18 | "name": "BasicAuthDetector" 19 | }, 20 | { 21 | "name": "CloudantDetector" 22 | }, 23 | { 24 | "name": "GitHubTokenDetector" 25 | }, 26 | { 27 | "name": "HexHighEntropyString", 28 | "limit": 3.0 29 | }, 30 | { 31 | "name": "IbmCloudIamDetector" 32 | }, 33 | { 34 | "name": "IbmCosHmacDetector" 35 | }, 36 | { 37 | "name": "JwtTokenDetector" 38 | }, 39 | { 40 | "name": "KeywordDetector", 41 | "keyword_exclude": "" 42 | }, 43 | { 44 | "name": "MailchimpDetector" 45 | }, 46 | { 47 | "name": "NpmDetector" 48 | }, 49 | { 50 | "name": "PrivateKeyDetector" 51 | }, 52 | { 53 | "name": "SendGridDetector" 54 | }, 55 | { 56 | "name": "SlackDetector" 57 | }, 58 | { 59 | "name": "SoftlayerDetector" 60 | }, 61 | { 62 | "name": "SquareOAuthDetector" 63 | }, 64 | { 65 | "name": "StripeDetector" 66 | }, 67 | { 68 | "name": "TwilioKeyDetector" 69 | } 70 | ], 71 | "filters_used": [ 72 | { 73 | "path": "detect_secrets.filters.allowlist.is_line_allowlisted" 74 | }, 75 | { 76 | "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", 77 | "min_level": 2 78 | }, 79 | { 80 | "path": "detect_secrets.filters.heuristic.is_indirect_reference" 81 | }, 82 | { 83 | "path": "detect_secrets.filters.heuristic.is_lock_file" 84 | }, 85 | { 86 | "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" 87 | }, 88 | { 89 | "path": "detect_secrets.filters.heuristic.is_potential_uuid" 90 | }, 91 | { 92 | "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" 93 | }, 94 | { 95 | "path": "detect_secrets.filters.heuristic.is_sequential_string" 96 | }, 97 | { 98 | "path": "detect_secrets.filters.heuristic.is_swagger_file" 99 | }, 100 | { 101 | "path": "detect_secrets.filters.heuristic.is_templated_secret" 102 | } 103 | ], 104 | "results": { 105 | "properties_test.go": [ 106 | { 107 | "type": "Base64 High Entropy String", 108 | "filename": "properties_test.go", 109 | "hashed_secret": "307ae3b4ea3d54d9ff0fdb36f9075c8d14a050c3", 110 | "is_verified": false, 111 | "line_number": 26 112 | } 113 | ] 114 | }, 115 | "generated_at": "2021-12-19T18:31:57Z" 116 | } 117 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: build tidy lint fmt test 2 | 3 | #------------------------------------------------------------------------- 4 | # Variables 5 | # ------------------------------------------------------------------------ 6 | env=CGO_ENABLED=1 7 | 8 | test: 9 | CGO_ENABLED=1 go test -v -cover -failfast -race ./... 10 | 11 | fuzz: 12 | @fuzzTime=$${FUZZ_TIME:-10}; \ 13 | files=$$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' .); \ 14 | for file in $$files; do \ 15 | funcs=$$(grep -o 'func Fuzz\w*' $$file | sed 's/func //'); \ 16 | for func in $$funcs; do \ 17 | echo "Fuzzing $$func in $$file"; \ 18 | parentDir=$$(dirname $$file); \ 19 | go test $$parentDir -run=$$func -fuzz=$$func -fuzztime=$${fuzzTime}s; \ 20 | if [ $$? -ne 0 ]; then \ 21 | echo "Fuzzing $$func in $$file failed"; \ 22 | exit 1; \ 23 | fi; \ 24 | done; \ 25 | done 26 | 27 | bench: 28 | go test -bench=. -benchmem ./... 29 | 30 | lint: 31 | golangci-lint run 32 | pre-commit run --all-files 33 | 34 | gomod2nix: 35 | gomod2nix generate 36 | 37 | build: gomod2nix test 38 | $(env) go build ./... 39 | 40 | release-dev: build-ci 41 | 42 | upgrade: 43 | pre-commit autoupdate 44 | go get -u ./... 45 | 46 | update: 47 | git submodule update --recursive 48 | 49 | fmt: 50 | gofmt -s -w . 51 | 52 | tidy: fmt 53 | go mod tidy 54 | 55 | clean: 56 | rm -rf dist 57 | rm -rf coverage 58 | 59 | #------------------------------------------------------------------------- 60 | # CI targets 61 | #------------------------------------------------------------------------- 62 | test-ci: 63 | CGO_ENABLED=1 go test \ 64 | -cover \ 65 | -covermode=atomic \ 66 | -coverprofile=coverage.txt \ 67 | -failfast \ 68 | -race ./... 69 | make fuzz FUZZ_TIME=10 70 | 71 | build-ci: test-ci 72 | $(env) go build ./... 73 | 74 | bench-ci: test-ci 75 | go test -bench=. ./... | tee output.txt 76 | 77 | release-ci: build-ci 78 | 79 | #------------------------------------------------------------------------- 80 | # Force targets 81 | #------------------------------------------------------------------------- 82 | 83 | FORCE: 84 | 85 | #------------------------------------------------------------------------- 86 | # Phony targets 87 | #------------------------------------------------------------------------- 88 | 89 | .PHONY: build test lint fuzz 90 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Atomizer - Massively Parallel Distributed Computing 2 | 3 | [![CI](https://github.com/devnw/atomizer/actions/workflows/build.yml/badge.svg)](https://github.com/devnw/atomizer/actions) 4 | [![Go Report Card](https://goreportcard.com/badge/go.atomizer.io/engine)](https://goreportcard.com/report/go.atomizer.io/engine) 5 | [![codecov](https://codecov.io/gh/devnw/atomizer/branch/main/graph/badge.svg)](https://codecov.io/gh/devnw/atomizer) 6 | [![Go Reference](https://pkg.go.dev/badge/go.atomizer.io/engine.svg)](https://pkg.go.dev/go.atomizer.io/engine) 7 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 8 | [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](http://makeapullrequest.com) 9 | 10 | Created to facilitate simplified construction of distributed systems, the 11 | Atomizer library was built with simplicity in mind. Exposing a simple API which 12 | allows users to create "Atoms" ([def](docs/definitions.md#atom)) that 13 | contain the atomic elements of process logic for an application, which, when 14 | paired with an "Electron"([def](docs/definitions.md#electron)) payload are 15 | executed in the Atomizer runtime. 16 | 17 | ## Index 18 | 19 | - [Atomizer - Massively Parallel Distributed Computing](#atomizer---massively-parallel-distributed-computing) 20 | - [Index](#index) 21 | - [Getting Started](#getting-started) 22 | - [Test App](#test-app) 23 | - [Conductor Creation](#conductor-creation) 24 | - [Atom Creation](#atom-creation) 25 | - [Electron Creation](#electron-creation) 26 | - [Properties - Atom Results](#properties---atom-results) 27 | - [Events](#events) 28 | - [Element Registration](#element-registration) 29 | - [Init Registration](#init-registration) 30 | - [Atomizer Instantiation Registration](#atomizer-instantiation-registration) 31 | - [Direct Registration](#direct-registration) 32 | 33 | ## Getting Started 34 | 35 | To use Atomizer you will first need to add it to your module. 36 | 37 | ```go 38 | go get -u go.atomizer.io/engine@latest 39 | ``` 40 | 41 | I highly recommend checking out the [Test App](#test-app) for a functional 42 | example of the Atomizer framework in action. 43 | 44 | To create an instance of Atomizer in your app you will need a Conductor 45 | ([def](docs/definitions.md#conductor)). Currently there is an AMQP conductor 46 | that was built for Atomizer which can be found 47 | here: [AMQP](https://github.com/devnw/amqp), or you can create 48 | your own conductor by following the 49 | [Conductor creation instructions](#conductor-creation). 50 | 51 | Once you have registered your Conductor using one of the 52 | [Element Registration](#element-registration) methods then you must create 53 | and register your Atom implementations following the 54 | [Atom Creation](#atom-creation) instructions. 55 | 56 | After registering at least one Conductor and one Atom in your Atomizer 57 | instance you can begin accepting requests. Here is an example of initializing 58 | the framework and registering an Atom and Conductor to begin processing. 59 | 60 | ```go 61 | 62 | // Initialize Atomizer 63 | a := Atomize(ctx, &MyConductor{}, &MyAtom{}) 64 | 65 | // Start the Atomizer processing system 66 | err := a.Exec() 67 | 68 | if err != nil { 69 | ... 70 | } 71 | ``` 72 | 73 | Now that the framework is initialized you can push processing requests 74 | through your Conductor for your registered Atoms and the Atomizer framework 75 | will execute the Electron([def](docs/definitions.md#electron)) payload by 76 | bonding it with the correct registered Atom and returning the resulting 77 | `Properties` over the Conductor back to the sender. 78 | 79 | ## Test App 80 | 81 | Since the concepts here are new to people seeing this for the first time a 82 | capstone team of students from the University of Illinois Springfield put 83 | together a test app to showcase a working implementation of the Atomizer 84 | framework. 85 | 86 | This Test App implements a MonteCarlo *π* simulation using the Atomizer 87 | framework. The Atom implementation can be found here: 88 | [MonteCarlo *π*](https://github.com/devnw/montecarlopi). This implementation 89 | uses two Atoms. The first Atom "MonteCarlo" is a 90 | [Spawner](docs/design-methodologies.md#atom-types) which creates payloads for 91 | the Toss Atom which is an [Atomic](docs/design-methodologies.md#atom-types) 92 | Atom. 93 | 94 | To run a simulation follow the steps laid out in [this blog post](https://benjiv.com/pi-day-special-2021/) 95 | which will describe how to pull down a copy of the Monte Carlo π docker 96 | containers running Atomizer. 97 | 98 | [Atomizer Test Console](https://github.com/devnw/atomizer-test-console) 99 | 100 | This test application will allow you to run the same MonteCarlo *π* 101 | simulation from command line without needing to setup a NodeJS app or pull 102 | down the corresponding Docker container. 103 | 104 | Thank you to 105 | 106 | - Matthew N Meyer ([@MatthewNormanMeyer](https://github.com/MatthewNormanMeyer)) 107 | - Megan Pugliese ([@mugatupotamus](https://github.com/mugatupotamus)) 108 | - Nick Heiting ([@nheiting](https://github.com/nheiting)) 109 | - Benji Vesterby ([@benjivesterby](https://github.com/benjivesterby)) 110 | 111 | ## Conductor Creation 112 | 113 | Creation of a conductor involves implementing the Conductor interface as 114 | described in the Atomizer library which can be seen below. 115 | 116 | ```go 117 | // Conductor is the interface that should be implemented for passing 118 | // electrons to the atomizer that need processing. This should generally be 119 | // registered with the atomizer in an initialization script 120 | type Conductor interface { 121 | 122 | // Receive gets the atoms from the source 123 | // that are available to atomize 124 | Receive(ctx context.Context) <-chan *Electron 125 | 126 | // Complete mark the completion of an electron instance 127 | // with applicable statistics 128 | Complete(ctx context.Context, p *Properties) error 129 | 130 | // Send sends electrons back out through the conductor for 131 | // additional processing 132 | Send(ctx context.Context, electron *Electron) (<-chan *Properties, error) 133 | 134 | // Close cleans up the conductor 135 | Close() 136 | } 137 | 138 | ``` 139 | 140 | Once you have created your Conductor you must then register it into the 141 | framework using one of the [Element Registration](#element-registration) 142 | methods for Atomizer. 143 | 144 | ## Atom Creation 145 | 146 | The Atomizer library is the framework on which you can build your distributed 147 | system. To do this you need to create "Atoms"([def](docs/definitions.md#atom)) 148 | which implement your specific business logic. To do this you must implement 149 | the Atom interface seen below. 150 | 151 | ```go 152 | type Atom interface { 153 | Process(ctx context.Context, c Conductor, e *Electron,) ([]byte, error) 154 | } 155 | ``` 156 | 157 | Once you have created your Atom you must then register it into the framework 158 | using one of the [Element Registration](#element-registration) methods for 159 | Atomizer. 160 | 161 | ## Electron Creation 162 | 163 | Electrons([def](docs/definitions.md#atom)) are one of the most important 164 | elements of the Atomizer framework because they supply the `data` necessary 165 | for the framework to properly execute an Atom since the Atom implementation 166 | is pure business logic. 167 | 168 | ```go 169 | // Electron is the base electron that MUST parse from the payload 170 | // from the conductor 171 | type Electron struct { 172 | // SenderID is the unique identifier for the node that sent the 173 | // electron 174 | SenderID string 175 | 176 | // ID is the unique identifier of this electron 177 | ID string 178 | 179 | // AtomID is the identifier of the atom for this electron instance 180 | // this is generally `package.Type`. Use the atomizer.ID() method 181 | // if unsure of the type for an Atom. 182 | AtomID string 183 | 184 | // Timeout is the maximum time duration that should be allowed 185 | // for this instance to process. After the duration is exceeded 186 | // the context should be canceled and the processing released 187 | // and a failure sent back to the conductor 188 | Timeout *time.Duration 189 | 190 | // CopyState lets atomizer know if it should copy the state of the 191 | // original atom registration to the new atom instance when processing 192 | // a newly received electron 193 | // 194 | // NOTE: Copying the state of an Atom as registered requires that ALL 195 | // fields that are to be copied are **EXPORTED** otherwise they are 196 | // skipped 197 | CopyState bool 198 | 199 | // Payload is to be used by the registered atom to properly unmarshal 200 | // the []byte for the actual atom instance. RawMessage is used to 201 | // delay unmarshal of the payload information so the atom can do it 202 | // internally 203 | Payload []byte 204 | } 205 | ``` 206 | 207 | The most important part for an Atom is the `Payload`. This `[]byte` holds data 208 | which can be read in your Atom. This is how the Atom receives state information 209 | for processing. Decoding the `Payload` is the responsibility of the Atom 210 | implementation. The other fields of the Electron are used by the Atomizer 211 | internally, but are available to an Atom as part of the Process method if 212 | necessary. 213 | 214 | Electrons are provided to the Atomizer framework through a registered 215 | Conductor, generally a Message Queue. 216 | 217 | ## Properties - Atom Results 218 | 219 | The results of Atom processing are contained in the `Properties` struct in 220 | the Atomizer library. This struct contains metadata about the processing that 221 | took place as well as the results or errors of the specific Atom which was 222 | executed from a request. 223 | 224 | ```go 225 | // Properties is the struct for storing properties information after the 226 | // processing of an atom has completed so that it can be sent to the 227 | // original requestor 228 | type Properties struct { 229 | ElectronID string 230 | AtomID string 231 | Start time.Time 232 | End time.Time 233 | Error error 234 | Result []byte 235 | } 236 | ``` 237 | 238 | The Result field is the `[]byte` that is returned from the Atom that was 239 | executed, and in general the Error property contains the `error` which was 240 | returned from the Atom as well, however if there are errors in processing 241 | an Atom that are not related to the internal processing of the Atom that 242 | error will be returned on the `Properties` struct in place of the Atom error 243 | as it is unlikely the Atom executed. 244 | 245 | ## Events 246 | 247 | Atomizer exports a method called `Events` which returns an 248 | `go.devnw.com/events.EventStream` and `Errors` which returns an 249 | `go.devnw.com/events.ErrorStream`. When you call either of 250 | these methods with a buffer of 0 they utilize an internal channel within the 251 | `go.devnw.com/event` package which then **MUST** be monitored for events/errors 252 | or it will block processing. 253 | 254 | The purpose of these methods is to allow for users to monitor events 255 | occurring inside of Atomizer. Because these use channels either pass a buffer 256 | value to the method or handle the channel in a `go` routine to keep it from 257 | blocking your application. 258 | 259 | Along with the `Events` and `Errors` methods, Atomizer exports two important 260 | structs. `atomizer.Error` and `atomizer.Event`. These contain information such 261 | as the `AtomID`, `ElectronID` or `ConductorID` that the event applies to as well 262 | as any message or error that may be part of the event. 263 | 264 | Both `atomizer.Error` and `atomizer.Event` implement the `fmt.Stringer` 265 | interface to make for easy logging. 266 | 267 | `atomizer.Error` also implements the `error` interface as well as the 268 | `Unwrap` method for nested errors. 269 | 270 | `atomizer.Event` also implements the `go.devnw.com/events.Event` interface to 271 | ensure the the `event` package can correctly handle the event. 272 | 273 | ```go 274 | // Event indicates an atomizer event has taken 275 | // place that is not categorized as an error 276 | // Event implements the stringer interface but 277 | // does NOT implement the error interface 278 | type Event struct { 279 | // Message from atomizer about this error 280 | Message string `json:"message"` 281 | 282 | // ElectronID is the associated electron instance 283 | // where the error occurred. Empty ElectronID indicates 284 | // the error was not part of a running electron instance. 285 | ElectronID string `json:"electronID"` 286 | 287 | // AtomID is the atom which was processing when 288 | // the error occurred. Empty AtomID indicates 289 | // the error was not part of a running atom. 290 | AtomID string `json:"atomID"` 291 | 292 | // ConductorID is the conductor which was being 293 | // used for receiving instructions 294 | ConductorID string `json:"conductorID"` 295 | } 296 | 297 | // Error is an error type which provides specific 298 | // atomizer information as part of an error 299 | type Error struct { 300 | 301 | // Event is the event that took place to create 302 | // the error and contains metadata relevant to the error 303 | Event *Event `json:"event"` 304 | 305 | // Internal is the internal error 306 | Internal error `json:"internal"` 307 | } 308 | ``` 309 | 310 | ## Element Registration 311 | 312 | There are three methods in Atomizer for registering `Atoms` and `Conductors`. 313 | 314 | - [Init Registration](#init-registration) 315 | - [Atomizer Instantiation Registration](#atomizer-instantiation-registration) 316 | - [Direct Registration](#direct-registration) 317 | 318 | ### Init Registration 319 | 320 | Atoms and Conductors can be registered in an `init` method in your code as seen 321 | below. 322 | 323 | ```go 324 | func init() { 325 | err := atomizer.Register(&MonteCarlo{}) 326 | if err != nil { 327 | ... 328 | } 329 | } 330 | ``` 331 | 332 | ### Atomizer Instantiation Registration 333 | 334 | Atoms and Conductors can be passed as the second argument to the `Atomize` 335 | method. This parameter is variadic and can accept *any* number of registrations. 336 | 337 | ```go 338 | a := Atomize(ctx, &MonteCarlo{}) 339 | ``` 340 | 341 | ### Direct Registration 342 | 343 | Direct registration happens when you use the `Register` method directly on an 344 | Atomizer instance. 345 | 346 | NOTE: The `Register` call can happen before or after the `a.Exec()` method call. 347 | 348 | ```go 349 | a := Atomize(ctx) 350 | 351 | // Start the Atomizer processing system 352 | err := a.Exec() 353 | 354 | if err != nil { 355 | ... 356 | } 357 | 358 | // Register the Atom 359 | err = a.Register(&MonteCarlo{}) 360 | if err != nil { 361 | ... 362 | } 363 | ``` 364 | -------------------------------------------------------------------------------- /atom.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "context" 10 | ) 11 | 12 | // Atom is an atomic action with process method for the atomizer to execute 13 | // the Atom 14 | type Atom interface { 15 | Process( 16 | ctx context.Context, 17 | conductor Conductor, 18 | electron *Electron, 19 | ) ([]byte, error) 20 | } 21 | -------------------------------------------------------------------------------- /atomizer.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "context" 10 | "reflect" 11 | "sync" 12 | "time" 13 | 14 | "github.com/mohae/deepcopy" 15 | "go.devnw.com/event" 16 | "go.devnw.com/validator" 17 | ) 18 | 19 | // atomizer facilitates the execution of tasks (aka Electrons) which 20 | // are received from the configured sources these electrons can be 21 | // distributed across many instances of the atomizer on different nodes 22 | // in a distributed system or in memory. Atoms should be created to 23 | // process "atomic" actions which are small in scope and overall processing 24 | // complexity minimizing time to run and allowing for the distributed 25 | // system to take on the burden of long running processes as a whole 26 | // rather than a single process handling the overall load 27 | type atomizer struct { 28 | 29 | // Electron Channel 30 | electrons chan instance 31 | 32 | // channel for passing the instance to a monitoring go routine 33 | bonded chan instance 34 | 35 | // This communicates the different conductors and atoms that are 36 | // registered into the system while it's alive 37 | registrations chan interface{} 38 | 39 | // This sync.Map contains the channels for handling each of the 40 | // bondings for the different atoms registered in the system 41 | atomsMu sync.RWMutex 42 | atoms map[string]chan<- instance 43 | 44 | publisher *event.Publisher 45 | 46 | ctx context.Context 47 | cancel context.CancelFunc 48 | 49 | execSyncOnce sync.Once 50 | } 51 | 52 | // Initialize the go routines that will read from the conductors concurrently 53 | // while other parts of the atomizer reads in the inputs and executes the 54 | // instances of electrons 55 | func (a *atomizer) receive() { 56 | if a.registrations == nil { 57 | a.publisher.ErrorFunc(a.ctx, func() error { 58 | return &Error{ 59 | Event: &Event{ 60 | Message: "nil registrations channel", 61 | }, 62 | } 63 | }) 64 | return 65 | } 66 | 67 | // TODO: Self-heal with heartbeats 68 | for { 69 | select { 70 | case <-a.ctx.Done(): 71 | return 72 | case r, ok := <-a.registrations: 73 | if !ok { 74 | a.publisher.ErrorFunc(a.ctx, func() error { 75 | return simple("registrations closed", nil) 76 | }) 77 | return 78 | } 79 | 80 | a.register(r) 81 | a.publisher.EventFunc(a.ctx, func() event.Event { 82 | return makeEvent("registered " + ID(r)) 83 | }) 84 | } 85 | } 86 | } 87 | 88 | // register the different receivable interfaces into the atomizer from 89 | // wherever they were sent from 90 | func (a *atomizer) register(input interface{}) { 91 | if !validator.Valid(input) { 92 | a.publisher.ErrorFunc(a.ctx, func() error { 93 | return simple("invalid registration "+ID(input), nil) 94 | }) 95 | } 96 | 97 | switch v := input.(type) { 98 | case Conductor: 99 | err := a.receiveConductor(v) 100 | if err == nil { 101 | a.publisher.EventFunc(a.ctx, func() event.Event { 102 | return &Event{ 103 | Message: "conductor received", 104 | ConductorID: ID(v), 105 | } 106 | }) 107 | } 108 | case Atom: 109 | 110 | err := a.receiveAtom(v) 111 | if err == nil { 112 | a.publisher.EventFunc(a.ctx, func() event.Event { 113 | return &Event{ 114 | Message: "atom received", 115 | AtomID: ID(v), 116 | } 117 | }) 118 | } 119 | default: 120 | a.publisher.ErrorFunc(a.ctx, func() error { 121 | return simple( 122 | "unknown registration type "+ID(input), 123 | nil, 124 | ) 125 | }) 126 | } 127 | } 128 | 129 | // receiveConductor setups a retrieval loop for the conductor 130 | func (a *atomizer) receiveConductor(conductor Conductor) error { 131 | if !validator.Valid(conductor) { 132 | return &Error{Event: &Event{ 133 | Message: "invalid conductor", 134 | ConductorID: ID(conductor), 135 | }} 136 | } 137 | 138 | go a.conduct(a.ctx, conductor) 139 | 140 | return nil 141 | } 142 | 143 | // conduct reads in from a specific electron channel of a conductor and drop 144 | // it onto the atomizer channel for electrons 145 | func (a *atomizer) conduct(ctx context.Context, conductor Conductor) { 146 | // Self Heal - Re-place the conductor on the register channel for 147 | // the atomizer to re-initialize so this stack can be 148 | // garbage collected 149 | 150 | // a.publisher.EventFunc(a.ctx, a.Register(conductor)) 151 | // })) 152 | 153 | receiver := conductor.Receive(ctx) 154 | 155 | // Read from the electron channel for a conductor and push onto 156 | // the a electron channel for processing 157 | for { 158 | select { 159 | case <-ctx.Done(): 160 | return 161 | case e, ok := <-receiver: 162 | if !ok { 163 | a.publisher.ErrorFunc(a.ctx, func() error { 164 | return &Error{Event: &Event{ 165 | Message: "receiver closed", 166 | ConductorID: ID(conductor), 167 | }} 168 | }) 169 | 170 | return 171 | } 172 | 173 | if !validator.Valid(e) { 174 | err := &Error{Event: &Event{ 175 | Message: "invalid electron", 176 | ElectronID: e.ID, 177 | ConductorID: ID(conductor), 178 | }} 179 | 180 | err.Internal = conductor.Complete( 181 | ctx, 182 | &Properties{ 183 | ElectronID: e.ID, 184 | AtomID: e.AtomID, 185 | Start: time.Now(), 186 | End: time.Now(), 187 | Error: err, 188 | Result: nil, 189 | }, 190 | ) 191 | 192 | a.publisher.ErrorFunc(a.ctx, func() error { 193 | return err 194 | }) 195 | 196 | continue 197 | } 198 | 199 | a.publisher.EventFunc(a.ctx, func() event.Event { 200 | return &Event{ 201 | Message: "electron received", 202 | ElectronID: e.ID, 203 | AtomID: e.AtomID, 204 | ConductorID: ID(conductor), 205 | } 206 | }) 207 | 208 | // Send the electron down the electrons 209 | // channel to be processed 210 | select { 211 | case <-a.ctx.Done(): 212 | return 213 | case a.electrons <- instance{ 214 | electron: e, 215 | conductor: conductor, 216 | }: 217 | a.publisher.EventFunc(a.ctx, func() event.Event { 218 | return &Event{ 219 | Message: "electron distributed", 220 | ElectronID: e.ID, 221 | AtomID: e.AtomID, 222 | ConductorID: ID(conductor), 223 | } 224 | }) 225 | } 226 | } 227 | } 228 | } 229 | 230 | // receiveAtom setups a retrieval loop for the conductor being passed in 231 | func (a *atomizer) receiveAtom(atom Atom) error { 232 | if !validator.Valid(atom) { 233 | return &Error{ 234 | Event: &Event{ 235 | Message: "invalid atom", 236 | AtomID: ID(atom), 237 | }, 238 | } 239 | } 240 | 241 | // Register the atom into the atomizer for receiving electrons 242 | a.atomsMu.Lock() 243 | defer a.atomsMu.Unlock() 244 | 245 | a.atoms[ID(atom)] = a.split(atom) 246 | a.publisher.EventFunc(a.ctx, func() event.Event { 247 | return &Event{ 248 | Message: "registered electron channel", 249 | AtomID: ID(atom), 250 | } 251 | }) 252 | 253 | return nil 254 | } 255 | 256 | func (a *atomizer) split(atom Atom) chan<- instance { 257 | electrons := make(chan instance) 258 | 259 | go a._split(atom, electrons) 260 | 261 | return electrons 262 | } 263 | 264 | func (a *atomizer) _split( 265 | atom Atom, 266 | electrons <-chan instance, 267 | ) { 268 | // Read from the electron channel for a conductor and push 269 | // onto the a electron channel for processing 270 | for { 271 | select { 272 | case <-a.ctx.Done(): 273 | return 274 | case inst, ok := <-electrons: 275 | if !ok { 276 | a.publisher.ErrorFunc(a.ctx, func() error { 277 | return &Error{ 278 | Event: &Event{ 279 | Message: "atom receiver closed", 280 | AtomID: ID(atom), 281 | }, 282 | } 283 | }) 284 | return 285 | } 286 | 287 | a.publisher.EventFunc(a.ctx, func() event.Event { 288 | return &Event{ 289 | Message: "new instance of electron", 290 | ElectronID: inst.electron.ID, 291 | AtomID: ID(atom), 292 | ConductorID: ID(inst.conductor), 293 | } 294 | }) 295 | 296 | // TODO: implement the processing push 297 | // TODO: after the processing has started 298 | // push to instances channel for monitoring 299 | // by the sampler so that this second can 300 | // focus on starting additional instances 301 | // rather than on individually bonded 302 | // instances 303 | 304 | var outatom Atom 305 | // Copy the state of the original registration to 306 | // the new atom 307 | if inst.electron.CopyState { 308 | outatom, _ = deepcopy.Copy(atom).(Atom) 309 | } else { 310 | // Initialize a new copy of the atom 311 | newAtom := reflect.New( 312 | reflect.TypeOf(atom).Elem(), 313 | ) 314 | 315 | // ok is not checked here because this should 316 | // never fail since the originating data item 317 | // is what created this 318 | outatom, _ = newAtom.Interface().(Atom) 319 | } 320 | 321 | a.exec(inst, outatom) 322 | } 323 | } 324 | } 325 | 326 | func (a *atomizer) exec(inst instance, atom Atom) { 327 | // bond the new atom instantiation to the electron instance 328 | if err := inst.bond(atom); err != nil { 329 | a.publisher.ErrorFunc(a.ctx, func() error { 330 | return &Error{ 331 | Event: &Event{ 332 | Message: "error while bonding", 333 | AtomID: ID(atom), 334 | ConductorID: ID(inst.conductor), 335 | }, 336 | Internal: err, 337 | } 338 | }) 339 | return 340 | } 341 | 342 | // Execute the instance after it's been 343 | // picked up for monitoring 344 | err := inst.execute(a.ctx) 345 | if err != nil { 346 | defer a.publisher.ErrorFunc(a.ctx, func() error { 347 | return &Error{ 348 | Internal: inst.properties.Error, 349 | Event: &Event{ 350 | Message: "error executing atom", 351 | AtomID: ID(atom), 352 | ElectronID: inst.electron.ID, 353 | }, 354 | } 355 | }) 356 | 357 | if inst.properties.Error != nil { 358 | inst.properties.Error = simple( 359 | "execution error", 360 | simple(err.Error(), 361 | simple( 362 | "instance error", 363 | inst.properties.Error, 364 | ), 365 | ), 366 | ) 367 | } else { 368 | inst.properties.Error = err 369 | } 370 | 371 | if inst.conductor != nil { 372 | completion := inst.conductor.Complete(a.ctx, inst.properties) 373 | a.publisher.ErrorFunc(a.ctx, func() error { 374 | return completion 375 | }) 376 | } 377 | } 378 | } 379 | 380 | func (a *atomizer) distribute() { 381 | for { 382 | select { 383 | case <-a.ctx.Done(): 384 | return 385 | case inst, ok := <-a.electrons: 386 | if !ok { 387 | a.publisher.ErrorFunc(a.ctx, func() error { 388 | return &Error{ 389 | Event: &Event{ 390 | Message: "dist channel closed", 391 | }, 392 | } 393 | }) 394 | 395 | return 396 | } 397 | 398 | a.atomsMu.RLock() 399 | achan, ok := a.atoms[inst.electron.AtomID] 400 | a.atomsMu.RUnlock() 401 | 402 | if !ok { 403 | // TODO: figure out what to do here 404 | // since the atom doesn't exist in 405 | // the registry 406 | 407 | a.publisher.ErrorFunc(a.ctx, func() error { 408 | return &Error{ 409 | Event: &Event{ 410 | Message: "not registered", 411 | AtomID: inst.electron.AtomID, 412 | ElectronID: inst.electron.ID, 413 | }, 414 | } 415 | }) 416 | continue 417 | } 418 | 419 | a.publisher.EventFunc(a.ctx, func() event.Event { 420 | return &Event{ 421 | Message: "pushing electron to atom", 422 | ElectronID: inst.electron.ID, 423 | AtomID: inst.electron.AtomID, 424 | ConductorID: ID(inst.conductor), 425 | } 426 | }) 427 | 428 | select { 429 | case <-a.ctx.Done(): 430 | return 431 | case achan <- inst: 432 | a.publisher.EventFunc(a.ctx, func() event.Event { 433 | return &Event{ 434 | Message: "pushed electron to atom", 435 | ElectronID: inst.electron.ID, 436 | AtomID: inst.electron.AtomID, 437 | ConductorID: ID(inst.conductor), 438 | } 439 | }) 440 | } 441 | } 442 | } 443 | } 444 | -------------------------------------------------------------------------------- /atomizer_exported.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | 12 | "go.devnw.com/event" 13 | "go.devnw.com/validator" 14 | ) 15 | 16 | // Atomizer interface implementation 17 | type Atomizer interface { 18 | Exec() error 19 | Register(value ...interface{}) error 20 | Wait() 21 | Events(buffer int) event.EventStream 22 | Errors(buffer int) event.ErrorStream 23 | 24 | // private methods enforce only this 25 | // package can return an atomizer 26 | isAtomizer() 27 | } 28 | 29 | // Atomize initialize instance of the atomizer to start reading from 30 | // conductors and execute bonded electrons/atoms 31 | // 32 | // NOTE: Registrations can be added through this method and OVERRIDE any 33 | // existing registrations of the same Atom or Conductor. 34 | func Atomize( 35 | ctx context.Context, 36 | registrations ...interface{}, 37 | ) (Atomizer, error) { 38 | err := Register(registrations...) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | ctx, cancel := _ctx(ctx) 44 | 45 | return &atomizer{ 46 | ctx: ctx, 47 | cancel: cancel, 48 | electrons: make(chan instance), 49 | bonded: make(chan instance), 50 | registrations: make(chan interface{}), 51 | atoms: make(map[string]chan<- instance), 52 | publisher: event.NewPublisher(ctx), 53 | }, nil 54 | } 55 | 56 | func (*atomizer) isAtomizer() {} 57 | 58 | // Events returns an go.devnw.com/event.EventStream for the atomizer 59 | func (a *atomizer) Events(buffer int) event.EventStream { return a.publisher.ReadEvents(buffer) } 60 | 61 | // Errors returns an go.devnw.com/event.ErrorStream for the atomizer 62 | func (a *atomizer) Errors(buffer int) event.ErrorStream { return a.publisher.ReadErrors(buffer) } 63 | 64 | // Exec kicks off the processing of the atomizer by pulling in the 65 | // pre-registrations through init calls on imported libraries and 66 | // starts up the receivers for atoms and conductors 67 | func (a *atomizer) Exec() (err error) { 68 | // Execute on the atomizer should only ever be run once 69 | a.execSyncOnce.Do(func() { 70 | defer a.publisher.EventFunc(a.ctx, func() event.Event { 71 | return makeEvent("pulling conductor and atom registrations") 72 | }) 73 | 74 | // Initialize the registrations in the Atomizer package 75 | for _, r := range Registrations() { 76 | a.register(r) 77 | } 78 | 79 | // Start up the receivers 80 | go a.receive() 81 | 82 | // Setup the distribution loop for incoming electrons 83 | // so that they can be properly fanned out to the 84 | // atom receivers 85 | go a.distribute() 86 | 87 | // TODO: Setup the instance receivers for monitoring of 88 | // individual instances as well as sending of outbound 89 | // electrons 90 | }) 91 | 92 | return err 93 | } 94 | 95 | // Register allows you to add additional type registrations to the atomizer 96 | // (ie. Conductors and Atoms) 97 | func (a *atomizer) Register(values ...interface{}) (err error) { 98 | defer func() { 99 | if r := recover(); r != nil { 100 | err = &Error{ 101 | Event: &Event{ 102 | Message: "panic in atomizer", 103 | }, 104 | Internal: ptoe(r), 105 | } 106 | } 107 | }() 108 | 109 | for _, value := range values { 110 | if !validator.Valid(value) { 111 | // TODO: create event here indicating that 112 | // a value was invalid and not registered 113 | continue 114 | } 115 | 116 | switch v := value.(type) { 117 | case Conductor, Atom: 118 | // Pass the value on the registrations 119 | // channel to be received 120 | select { 121 | case <-a.ctx.Done(): 122 | return simple("context closed", nil) 123 | case a.registrations <- v: 124 | } 125 | default: 126 | return simple( 127 | fmt.Sprintf( 128 | "invalid value in registration %s", 129 | ID(value), 130 | ), 131 | nil, 132 | ) 133 | } 134 | } 135 | 136 | return err 137 | } 138 | 139 | // Wait blocks on the context done channel to allow for the executable 140 | // to block for the atomizer to finish processing 141 | func (a *atomizer) Wait() { 142 | <-a.ctx.Done() 143 | } 144 | -------------------------------------------------------------------------------- /atomizer_mock_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "sync" 8 | "testing" 9 | 10 | "github.com/Pallinder/go-randomdata" 11 | "github.com/google/uuid" 12 | "github.com/pkg/errors" 13 | "go.devnw.com/alog" 14 | "go.devnw.com/event" 15 | "go.devnw.com/validator" 16 | ) 17 | 18 | type tresult struct { 19 | result string 20 | electron *Electron 21 | // err bool 22 | // panic bool 23 | } 24 | 25 | var noopelectron = &Electron{ 26 | SenderID: "empty", 27 | ID: "empty", 28 | AtomID: "empty", 29 | } 30 | 31 | var noopinvalidelectron = &Electron{} 32 | 33 | type invalidconductor struct{} 34 | 35 | type noopconductor struct{} 36 | 37 | func (*noopconductor) Receive(ctx context.Context) <-chan *Electron { 38 | return nil 39 | } 40 | 41 | func (*noopconductor) Send( 42 | ctx context.Context, 43 | electron *Electron, 44 | ) (<-chan *Properties, error) { 45 | return nil, nil 46 | } 47 | 48 | func (*noopconductor) Close() {} 49 | 50 | func (*noopconductor) Complete( 51 | ctx context.Context, 52 | properties *Properties, 53 | ) error { 54 | return nil 55 | } 56 | 57 | type noopatom struct{} 58 | 59 | func (*noopatom) Process( 60 | ctx context.Context, 61 | conductor Conductor, 62 | electron *Electron, 63 | ) ([]byte, error) { 64 | return nil, nil 65 | } 66 | 67 | type panicatom struct{} 68 | 69 | func (*panicatom) Process( 70 | ctx context.Context, 71 | conductor Conductor, 72 | electron *Electron, 73 | ) ([]byte, error) { 74 | panic("test panic") 75 | } 76 | 77 | type invalidatom struct{} 78 | 79 | func (*invalidatom) Process( 80 | ctx context.Context, 81 | conductor Conductor, 82 | electron *Electron, 83 | ) ([]byte, error) { 84 | return nil, nil 85 | } 86 | 87 | func (*invalidatom) Validate() bool { 88 | return false 89 | } 90 | 91 | type validconductor struct { 92 | echan chan *Electron 93 | valid bool 94 | } 95 | 96 | func (cond *validconductor) Receive(ctx context.Context) <-chan *Electron { 97 | return cond.echan 98 | } 99 | 100 | func (cond *validconductor) Send( 101 | ctx context.Context, 102 | electron *Electron, 103 | ) (response <-chan *Properties, err error) { 104 | return response, err 105 | } 106 | 107 | func (cond *validconductor) Validate() (valid bool) { 108 | return cond.valid && cond.echan != nil 109 | } 110 | 111 | func (cond *validconductor) Complete( 112 | ctx context.Context, 113 | properties *Properties, 114 | ) (err error) { 115 | return err 116 | } 117 | 118 | func (cond *validconductor) Close() {} 119 | 120 | // TODO: Move passthrough as a conductor implementation for in-node processing 121 | type passthrough struct { 122 | input chan *Electron 123 | results sync.Map 124 | } 125 | 126 | func (pt *passthrough) Receive(ctx context.Context) <-chan *Electron { 127 | return pt.input 128 | } 129 | 130 | func (pt *passthrough) Validate() bool { return pt.input != nil } 131 | 132 | func (pt *passthrough) Complete(ctx context.Context, p *Properties) error { 133 | if !validator.Valid(p) { 134 | return errors.Errorf( 135 | "invalid properties returned for electron [%s]", 136 | p.ElectronID, 137 | ) 138 | } 139 | 140 | // for rabbit mq drop properties onto the /basepath/electronid message path 141 | value, ok := pt.results.Load(p.ElectronID) 142 | if !ok { 143 | return errors.Errorf( 144 | "unable to load properties channel from sync map for electron [%s]", 145 | p.ElectronID, 146 | ) 147 | } 148 | 149 | if value == nil { 150 | return errors.Errorf( 151 | "nil properties channel returned for electron [%s]", 152 | p.ElectronID, 153 | ) 154 | } 155 | 156 | resultChan, ok := value.(chan *Properties) 157 | if !ok { 158 | return errors.New("unable to type assert electron properties channel") 159 | } 160 | 161 | defer close(resultChan) 162 | 163 | // Push the properties of the instance onto the channel 164 | select { 165 | case <-ctx.Done(): 166 | return nil 167 | case resultChan <- p: 168 | } 169 | return nil 170 | } 171 | 172 | func (pt *passthrough) Send( 173 | ctx context.Context, 174 | electron *Electron, 175 | ) (<-chan *Properties, error) { 176 | var err error 177 | result := make(chan *Properties) 178 | 179 | go func(result chan *Properties) { 180 | // Only kick off the electron for processing if there isn't already an 181 | // instance loaded in the system 182 | if _, loaded := pt.results.LoadOrStore(electron.ID, result); !loaded { 183 | // Push the electron onto the input channel 184 | select { 185 | case <-ctx.Done(): 186 | return 187 | case pt.input <- electron: 188 | // setup a monitoring thread for /basepath/electronid 189 | } 190 | } else { 191 | defer close(result) 192 | p := &Properties{} 193 | alog.Errorf(nil, "duplicate electron registration for EID [%s]", electron.ID) 194 | 195 | result <- p 196 | } 197 | }(result) 198 | 199 | return result, err 200 | } 201 | 202 | func (pt *passthrough) Close() {} 203 | 204 | type printer struct{ t *testing.T } 205 | 206 | type state struct{ ID string } 207 | 208 | func (s *state) Process(ctx context.Context, conductor Conductor, electron *Electron) (result []byte, err error) { 209 | return []byte(s.ID), nil 210 | } 211 | 212 | func (p *printer) Process(ctx context.Context, conductor Conductor, electron *Electron) (result []byte, err error) { 213 | if validator.Valid(electron) { 214 | var payload printerdata 215 | 216 | if err = json.Unmarshal(electron.Payload, &payload); err == nil { 217 | p.t.Logf("message from electron [%s] is: %s\n", electron.ID, payload.Message) 218 | } 219 | } 220 | 221 | return result, err 222 | } 223 | 224 | type returner struct{} 225 | 226 | func (b *returner) Process(ctx context.Context, conductor Conductor, electron *Electron) (result []byte, err error) { 227 | if !validator.Valid(electron) { 228 | return nil, errors.New("invalid electron") 229 | } 230 | 231 | var payload = &printerdata{} 232 | err = json.Unmarshal(electron.Payload, payload) 233 | if err != nil { 234 | return nil, err 235 | } 236 | 237 | result = []byte(payload.Message) 238 | alog.Println("returning payload") 239 | 240 | return result, err 241 | } 242 | 243 | func spawnReturner(size int) (tests []*tresult) { 244 | tests = make([]*tresult, 0, size) 245 | 246 | for i := 0; i < size; i++ { 247 | msg := randomdata.SillyName() 248 | 249 | e := newElectron( 250 | ID(returner{}), 251 | []byte(fmt.Sprintf("{\"message\":%q}", msg)), 252 | ) 253 | 254 | tests = append(tests, &tresult{ 255 | result: msg, 256 | electron: e, 257 | }) 258 | } 259 | 260 | return tests 261 | } 262 | 263 | type printerdata struct { 264 | Message string `json:"message"` 265 | } 266 | 267 | func newElectron(atomID string, payload []byte) *Electron { 268 | return &Electron{ 269 | SenderID: uuid.New().String(), 270 | ID: uuid.New().String(), 271 | AtomID: atomID, 272 | Payload: payload, 273 | } 274 | } 275 | 276 | // harness creates a valid atomizer that uses the passthrough conductor 277 | func harness( 278 | ctx context.Context, 279 | buffer int, 280 | atoms ...Atom, 281 | ) (Conductor, event.EventStream, error) { 282 | pass := &passthrough{ 283 | input: make(chan *Electron, 1), 284 | } 285 | 286 | // Register the conductor so it's picked up 287 | // when the atomizer is initialized 288 | if err := Register(pass); err != nil { 289 | return nil, nil, err 290 | } 291 | 292 | // Test Atom registrations 293 | 294 | if err := Register(&printer{}); err != nil { 295 | return nil, nil, err 296 | } 297 | 298 | if err := Register(&noopatom{}); err != nil { 299 | return nil, nil, err 300 | } 301 | 302 | if err := Register(&returner{}); err != nil { 303 | return nil, nil, err 304 | } 305 | 306 | for _, a := range atoms { 307 | if err := Register(a); err != nil { 308 | return nil, nil, err 309 | } 310 | } 311 | 312 | // Initialize the atomizer 313 | mizer, err := Atomize(ctx) 314 | if err != nil { 315 | return nil, nil, fmt.Errorf("error creating atomizer | %s", err) 316 | } 317 | 318 | a, _ := mizer.(*atomizer) 319 | 320 | var events event.EventStream 321 | if buffer >= 0 { 322 | events = a.Events(buffer) 323 | } 324 | 325 | // Start the execution threads 326 | return pass, events, a.Exec() 327 | } 328 | -------------------------------------------------------------------------------- /atomizer_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "testing" 8 | "time" 9 | 10 | "github.com/Pallinder/go-randomdata" 11 | "github.com/google/uuid" 12 | "go.devnw.com/event" 13 | "go.devnw.com/validator" 14 | ) 15 | 16 | func printEvents( 17 | ctx context.Context, 18 | t *testing.T, 19 | events event.EventStream, 20 | ) { 21 | for { 22 | select { 23 | case <-ctx.Done(): 24 | return 25 | case e, ok := <-events: 26 | if ok { 27 | t.Log(e) 28 | } else { 29 | return 30 | } 31 | } 32 | } 33 | } 34 | 35 | func TestAtomizer_Atomize_Register_Fail(t *testing.T) { 36 | // Register invalid atom 37 | _, err := Atomize( 38 | context.TODO(), 39 | nil, 40 | &struct{}{}, 41 | ) 42 | if err == nil { 43 | t.Fatalf("expected error | %s", err) 44 | } 45 | } 46 | 47 | func TestAtomizer_Exec(t *testing.T) { 48 | d := time.Second * 30 49 | // Setup a cancellation context for the test 50 | ctx, cancel := _ctxT(context.TODO(), &d) 51 | defer cancel() 52 | 53 | // Execute clean at beginning and end 54 | reset(ctx, t) 55 | t.Cleanup(func() { 56 | reset(context.TODO(), t) 57 | }) 58 | 59 | t.Log("setting up harness") 60 | conductor, events, err := harness(ctx, 1000) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | 65 | t.Log("setting up printing of atomizer events") 66 | go printEvents(ctx, t, events) 67 | 68 | t.Log("creating test electron") 69 | msg := randomdata.SillyName() 70 | e := newElectron( 71 | ID(returner{}), 72 | []byte( 73 | fmt.Sprintf("{\"message\":%q}", msg), 74 | ), 75 | ) 76 | 77 | test := &tresult{ 78 | result: msg, 79 | electron: e, 80 | } 81 | 82 | var sent = time.Now() 83 | 84 | t.Log("sending electron through conductor") 85 | // Send the electron onto the conductor 86 | result, err := conductor.Send(ctx, test.electron) 87 | if err != nil { 88 | t.Fatal(err) 89 | } 90 | 91 | t.Log("read result from passthrough conductor") 92 | check(ctx, t, test, e, result) 93 | 94 | t.Logf( 95 | "Processing Time Through Atomizer %s\n", 96 | time.Since(sent).String(), 97 | ) 98 | } 99 | 100 | func check( 101 | ctx context.Context, 102 | t *testing.T, 103 | test *tresult, 104 | e *Electron, 105 | result <-chan *Properties, 106 | ) { 107 | // Block until a result is returned from the instance 108 | select { 109 | case <-ctx.Done(): 110 | t.Fatal("context closed, test failed") 111 | return 112 | case result, ok := <-result: 113 | if !ok { 114 | t.Fatal("result channel closed, test failed") 115 | } 116 | 117 | if result.Error != nil { 118 | t.Fatal("Errors returned from atom", e) 119 | } 120 | 121 | if len(result.Result) == 0 { 122 | t.Fatal("results length is not 1") 123 | } 124 | 125 | res := string(result.Result) 126 | if res != test.result { 127 | t.Fatalf("%s != %s", test.result, res) 128 | } 129 | 130 | t.Logf( 131 | "EID [%s] | Time [%s] - MATCH", 132 | result.ElectronID, 133 | result.End.Sub(result.Start).String(), 134 | ) 135 | } 136 | } 137 | 138 | func TestAtomizer_initReg_Exec(t *testing.T) { 139 | d := time.Second * 30 140 | // Setup a cancellation context for the test 141 | ctx, cancel := _ctxT(context.TODO(), &d) 142 | defer cancel() 143 | 144 | // Execute clean at beginning and end 145 | reset(ctx, t) 146 | t.Cleanup(func() { 147 | reset(context.TODO(), t) 148 | }) 149 | 150 | t.Log("setting up harness") 151 | conductor, events, err := harness(ctx, 1000) 152 | if err != nil { 153 | t.Fatal(err) 154 | } 155 | 156 | t.Log("setting up printing of atomizer events") 157 | go printEvents(ctx, t, events) 158 | 159 | t.Log("creating test electron") 160 | msg := randomdata.SillyName() 161 | e := newElectron( 162 | ID(returner{}), 163 | []byte( 164 | fmt.Sprintf("{\"message\":%q}", msg), 165 | ), 166 | ) 167 | 168 | test := &tresult{ 169 | result: msg, 170 | electron: e, 171 | } 172 | 173 | var sent = time.Now() 174 | 175 | t.Log("sending electron through conductor") 176 | // Send the electron onto the conductor 177 | result, err := conductor.Send(ctx, test.electron) 178 | if err != nil { 179 | t.Fatal(err) 180 | } 181 | 182 | t.Log("read result from passthrough conductor") 183 | check(ctx, t, test, e, result) 184 | 185 | t.Logf( 186 | "Processing Time Through Atomizer %s\n", 187 | time.Since(sent).String(), 188 | ) 189 | } 190 | 191 | func TestAtomizer_Copy_State(t *testing.T) { 192 | d := time.Second * 30 193 | // Setup a cancellation context for the test 194 | ctx, cancel := _ctxT(context.TODO(), &d) 195 | defer cancel() 196 | 197 | // Execute clean at beginning and end 198 | reset(ctx, t) 199 | t.Cleanup(func() { 200 | reset(context.TODO(), t) 201 | }) 202 | 203 | stateid := uuid.New().String() 204 | 205 | t.Log("setting up harness") 206 | conductor, events, err := harness(ctx, 1000, &state{stateid}) 207 | if err != nil { 208 | t.Fatal(err) 209 | } 210 | 211 | t.Log("setting up printing of atomizer events") 212 | go printEvents(ctx, t, events) 213 | 214 | t.Log("creating test electron") 215 | e := &Electron{ 216 | SenderID: uuid.New().String(), 217 | ID: uuid.New().String(), 218 | AtomID: ID(state{}), 219 | CopyState: true, 220 | } 221 | 222 | var sent = time.Now() 223 | 224 | t.Log("sending electron through conductor") 225 | // Send the electron onto the conductor 226 | result, err := conductor.Send(ctx, e) 227 | if err != nil { 228 | t.Fatal(err) 229 | } 230 | 231 | t.Log("read result from passthrough conductor") 232 | // Block until a result is returned from the instance 233 | select { 234 | case <-ctx.Done(): 235 | t.Fatal("context closed, test failed") 236 | return 237 | case result, ok := <-result: 238 | if !ok { 239 | t.Fatal("result channel closed, test failed") 240 | } 241 | 242 | if result.Error != nil { 243 | t.Fatal("Errors returned from atom", e) 244 | } 245 | 246 | res := string(result.Result) 247 | if res != stateid { 248 | t.Fatalf("[%s] != [%s]", res, stateid) 249 | } 250 | 251 | t.Logf( 252 | "EID [%s] | Time [%s] - MATCH", 253 | result.ElectronID, 254 | result.End.Sub(result.Start).String(), 255 | ) 256 | } 257 | 258 | t.Logf( 259 | "Processing Time Through Atomizer %s\n", 260 | time.Since(sent).String(), 261 | ) 262 | } 263 | 264 | func TestAtomizer_Copy_State_Disabled(t *testing.T) { 265 | d := time.Second * 30 266 | // Setup a cancellation context for the test 267 | ctx, cancel := _ctxT(context.TODO(), &d) 268 | defer cancel() 269 | 270 | // Execute clean at beginning and end 271 | reset(ctx, t) 272 | t.Cleanup(func() { 273 | reset(context.TODO(), t) 274 | }) 275 | 276 | stateid := uuid.New().String() 277 | 278 | t.Log("setting up harness") 279 | conductor, events, err := harness(ctx, 1000, &state{stateid}) 280 | if err != nil { 281 | t.Fatal(err) 282 | } 283 | 284 | t.Log("setting up printing of atomizer events") 285 | go printEvents(ctx, t, events) 286 | 287 | t.Log("creating test electron") 288 | e := &Electron{ 289 | SenderID: uuid.New().String(), 290 | ID: uuid.New().String(), 291 | AtomID: ID(state{}), 292 | CopyState: false, 293 | } 294 | 295 | var sent = time.Now() 296 | 297 | t.Log("sending electron through conductor") 298 | // Send the electron onto the conductor 299 | result, err := conductor.Send(ctx, e) 300 | if err != nil { 301 | t.Fatal(err) 302 | } 303 | 304 | t.Log("read result from passthrough conductor") 305 | // Block until a result is returned from the instance 306 | select { 307 | case <-ctx.Done(): 308 | t.Fatal("context closed, test failed") 309 | return 310 | case result, ok := <-result: 311 | if !ok { 312 | t.Fatal("result channel closed, test failed") 313 | } 314 | 315 | if result.Error != nil { 316 | t.Fatal("Errors returned from atom", e) 317 | } 318 | 319 | res := string(result.Result) 320 | if res == stateid { 321 | t.Fatalf("Expected mismatch [%s] == [%s]", res, stateid) 322 | } 323 | 324 | t.Logf( 325 | "EID [%s] | Time [%s] - MATCH", 326 | result.ElectronID, 327 | result.End.Sub(result.Start).String(), 328 | ) 329 | } 330 | 331 | t.Logf( 332 | "Processing Time Through Atomizer %s\n", 333 | time.Since(sent).String(), 334 | ) 335 | } 336 | 337 | func TestAtomizer_Exec_Returner(t *testing.T) { 338 | ctx, cancel := _ctx(context.TODO()) 339 | defer cancel() 340 | 341 | reset(ctx, t) 342 | t.Cleanup(func() { 343 | reset(context.TODO(), t) 344 | }) 345 | 346 | t.Log("Initializing Test Harness") 347 | 348 | conductor, _, err := harness(ctx, -1) 349 | if err != nil { 350 | t.Fatalf("error while executing harness | %s", err) 351 | } 352 | 353 | t.Log("Harness Successfully Created") 354 | 355 | var sent = time.Now() 356 | wg := sync.WaitGroup{} 357 | 358 | tests := spawnReturner(50) 359 | 360 | t.Logf("[%v] tests loaded", len(tests)) 361 | 362 | results := make(chan Properties) 363 | 364 | go func() { 365 | defer cancel() 366 | 367 | for _, test := range tests { 368 | wg.Add(1) 369 | go func(test *tresult) { 370 | defer wg.Done() 371 | 372 | sentAndEval( 373 | ctx, 374 | t, 375 | conductor, 376 | test, 377 | ) 378 | }(test) 379 | } 380 | 381 | wg.Wait() 382 | close(results) 383 | }() 384 | 385 | <-ctx.Done() 386 | t.Logf( 387 | "Processing Time Through Atomizer %s\n", 388 | time.Since(sent).String(), 389 | ) 390 | } 391 | 392 | func sentAndEval( 393 | ctx context.Context, 394 | t *testing.T, 395 | c Conductor, 396 | test *tresult, 397 | ) { 398 | // Send the electron onto the conductor 399 | result, err := c.Send(ctx, test.electron) 400 | if err != nil { 401 | t.Fatalf("Error sending electron %s", test.electron.ID) 402 | } 403 | 404 | select { 405 | case <-ctx.Done(): 406 | return 407 | case result, ok := <-result: 408 | if !ok { 409 | t.Fatal("result channel closed, test failed") 410 | } 411 | 412 | if result.Error != nil { 413 | t.Fatal( 414 | "Error returned from atom", 415 | result.Error, 416 | ) 417 | } 418 | 419 | if !validator.Valid(result.Result) { 420 | t.Fatal("results length is not 1") 421 | } 422 | 423 | res := string(result.Result) 424 | if res != test.result { 425 | t.Fatalf("%s != %s", test.result, res) 426 | } 427 | } 428 | } 429 | 430 | // Tests the atomizer creation method without a conductor 431 | func TestAtomizeNoConductors(t *testing.T) { 432 | tests := []struct { 433 | key string 434 | value interface{} 435 | err bool 436 | }{ 437 | { 438 | "ValidTestEmptyConductor", 439 | nil, 440 | false, 441 | }, 442 | { 443 | "ValidTestValidConductor", 444 | &validconductor{make(chan *Electron), true}, 445 | false, 446 | }, 447 | { 448 | "InvalidTestInvalidConductor", 449 | &invalidconductor{}, 450 | true, 451 | }, 452 | { 453 | "InvalidTestNilConductor", 454 | nil, 455 | true, 456 | }, 457 | { 458 | "InvalidTestInvalidElectronChan", 459 | &validconductor{}, 460 | true, 461 | }, 462 | } 463 | 464 | ctx, cancel := _ctx(context.TODO()) 465 | defer cancel() 466 | 467 | for _, test := range tests { 468 | t.Run(test.key, func(t *testing.T) { 469 | reset(ctx, t) 470 | t.Cleanup(func() { 471 | reset(context.TODO(), t) 472 | }) 473 | 474 | // Store the test conductor 475 | if test.err || (!test.err && test.value != nil) { 476 | // TODO: should the error be ignored here? 477 | // Store invalid conductor 478 | _ = Register(test.value) 479 | } 480 | 481 | a, err := Atomize(ctx) 482 | if err != nil { 483 | t.Fatal(err) 484 | } 485 | 486 | err = a.Exec() 487 | if err != nil { 488 | t.Fatal(err) 489 | } 490 | 491 | if !validator.Valid(a) { 492 | t.Fatalf("atomizer was expected to be valid but was returned invalid") 493 | } 494 | }) 495 | } 496 | } 497 | 498 | func TestAtomizer_AddConductor(t *testing.T) { 499 | ctx, cancel := _ctx(context.TODO()) 500 | defer cancel() 501 | 502 | tests := []struct { 503 | key string 504 | value interface{} 505 | err bool 506 | }{ 507 | { 508 | "ValidTestEmptyConductor", 509 | &validconductor{make(chan *Electron), true}, 510 | false, 511 | }, 512 | { 513 | "InvalidTestConductor", 514 | &validconductor{make(chan *Electron), false}, 515 | true, 516 | }, 517 | { 518 | "InvalidTestNilConductor", 519 | nil, 520 | true, 521 | }, 522 | { 523 | "InvalidTestInvalidElectronChan", 524 | &validconductor{}, 525 | true, 526 | }, 527 | } 528 | 529 | for _, test := range tests { 530 | t.Run(test.key, func(t *testing.T) { 531 | // Reset sync map for this test 532 | reset(ctx, t) 533 | t.Cleanup(func() { 534 | reset(context.TODO(), t) 535 | }) 536 | 537 | a, err := Atomize(ctx) 538 | if err != nil { 539 | t.Fatal(err) 540 | } 541 | 542 | err = a.Exec() 543 | if err != nil { 544 | t.Fatal(err) 545 | } 546 | 547 | // Add the conductor 548 | err = a.Register(test.value) 549 | if err != nil && !test.err { 550 | t.Fatalf("expected success, received error") 551 | } 552 | }) 553 | } 554 | } 555 | 556 | func TestAtomizer_register_Errs(t *testing.T) { 557 | ctx, cancel := _ctx(context.TODO()) 558 | defer cancel() 559 | 560 | tests := []struct { 561 | key string 562 | a *atomizer 563 | value interface{} 564 | }{ 565 | { 566 | "invalid conductor test", 567 | &atomizer{ 568 | ctx: ctx, 569 | publisher: event.NewPublisher(ctx), 570 | }, 571 | &validconductor{}, 572 | }, 573 | { 574 | "Invalid Struct Type", 575 | &atomizer{ 576 | ctx: ctx, 577 | publisher: event.NewPublisher(ctx), 578 | }, 579 | &struct{}{}, 580 | }, 581 | } 582 | 583 | for _, test := range tests { 584 | t.Run(test.key, func(t *testing.T) { 585 | errors := test.a.Errors(1) 586 | 587 | test.a.register(test.value) 588 | 589 | out, ok := <-errors 590 | if !ok { 591 | t.Fatal("channel closed") 592 | } 593 | 594 | t.Log(out) 595 | }) 596 | } 597 | } 598 | 599 | func TestAtomizer_Register_Errs(t *testing.T) { 600 | ctx, cancel := _ctx(context.TODO()) 601 | cancel() 602 | 603 | tests := []struct { 604 | key string 605 | a *atomizer 606 | value interface{} 607 | }{ 608 | { 609 | "panic test, nil channels", 610 | &atomizer{publisher: event.NewPublisher(ctx)}, 611 | &validconductor{make(chan *Electron), true}, 612 | }, 613 | { 614 | "close context test", 615 | &atomizer{ctx: ctx, publisher: event.NewPublisher(ctx)}, 616 | &validconductor{make(chan *Electron), true}, 617 | }, 618 | { 619 | "Invalid Struct Type", 620 | &atomizer{publisher: event.NewPublisher(ctx)}, 621 | &struct{}{}, 622 | }, 623 | } 624 | 625 | for _, test := range tests { 626 | t.Run(test.key, func(t *testing.T) { 627 | // Add the conductor 628 | err := test.a.Register(test.value) 629 | if err == nil { 630 | t.Fatalf("expected error, received success") 631 | } 632 | }) 633 | } 634 | } 635 | 636 | func TestAtomizer_receive_panic(t *testing.T) { 637 | a := &atomizer{ 638 | ctx: context.Background(), 639 | publisher: event.NewPublisher(context.Background()), 640 | } 641 | 642 | errors := a.Errors(1) 643 | 644 | defer func() { 645 | r := recover() 646 | if r != nil { 647 | t.Fatal("unexpected panic") 648 | } 649 | }() 650 | 651 | a.receive() 652 | 653 | err, ok := <-errors 654 | if !ok || err == nil { 655 | t.Fatal("channel closed") 656 | } 657 | } 658 | 659 | func TestAtomizer_receive_nilreg(t *testing.T) { 660 | a := &atomizer{ 661 | ctx: context.Background(), 662 | publisher: event.NewPublisher(context.Background()), 663 | } 664 | 665 | errors := a.Errors(1) 666 | 667 | a.receive() 668 | 669 | _, ok := <-errors 670 | if !ok { 671 | t.Fatal("channel closed") 672 | } 673 | } 674 | 675 | func TestAtomizer_receive_closedReg(t *testing.T) { 676 | a := &atomizer{ 677 | ctx: context.Background(), 678 | registrations: make(chan interface{}), 679 | publisher: event.NewPublisher(context.Background()), 680 | } 681 | 682 | close(a.registrations) 683 | 684 | errors := a.Errors(1) 685 | 686 | a.receive() 687 | 688 | _, ok := <-errors 689 | if !ok { 690 | t.Fatal("channel closed") 691 | } 692 | } 693 | 694 | func TestAtomizer_receiveAtom_invalid(t *testing.T) { 695 | a := &atomizer{publisher: event.NewPublisher(context.Background())} 696 | 697 | err := a.receiveAtom(&invalidatom{}) 698 | if err == nil { 699 | t.Fatal("expected error") 700 | } 701 | } 702 | 703 | func TestAtomizer_conduct_closedreceiver(t *testing.T) { 704 | c := &validconductor{echan: make(chan *Electron)} 705 | close(c.echan) 706 | 707 | a := &atomizer{ 708 | ctx: context.Background(), 709 | publisher: event.NewPublisher(context.Background()), 710 | } 711 | 712 | errors := a.Errors(1) 713 | 714 | a.conduct(context.Background(), c) 715 | 716 | _, ok := <-errors 717 | if !ok { 718 | t.Fatal("channel closed") 719 | } 720 | } 721 | 722 | func TestAtomizer_conduct_panic(t *testing.T) { 723 | c := &validconductor{echan: make(chan *Electron)} 724 | close(c.echan) 725 | 726 | a := &atomizer{publisher: event.NewPublisher(context.Background())} 727 | 728 | errors := a.Errors(2) 729 | 730 | defer func() { 731 | r := recover() 732 | if r != nil { 733 | t.Fatal("unexpected panic") 734 | } 735 | }() 736 | 737 | a.conduct(context.Background(), c) 738 | 739 | err, ok := <-errors 740 | if !ok || err == nil { 741 | t.Fatal("expected error") 742 | } 743 | } 744 | 745 | func TestAtomizer_conduct_invalidE(t *testing.T) { 746 | c := &passthrough{input: make(chan *Electron)} 747 | a := &atomizer{ 748 | ctx: context.Background(), 749 | publisher: event.NewPublisher(context.Background()), 750 | } 751 | go a.conduct(context.Background(), c) 752 | 753 | t.Log("sending") 754 | results, err := c.Send(context.Background(), noopinvalidelectron) 755 | if err != nil { 756 | t.Fatal(err) 757 | } 758 | 759 | t.Log("waiting on results") 760 | res, ok := <-results 761 | if !ok { 762 | t.Fatal("unexpected closed channel") 763 | } 764 | 765 | if res.Error == nil { 766 | t.Fatal("expected error result") 767 | } 768 | } 769 | 770 | func TestAtomizer_split_closedEchan(t *testing.T) { 771 | a := &atomizer{ 772 | ctx: context.Background(), 773 | publisher: event.NewPublisher(context.Background()), 774 | } 775 | 776 | errors := a.Errors(1) 777 | 778 | echan := make(chan instance) 779 | close(echan) 780 | 781 | a._split(nil, echan) 782 | 783 | _, ok := <-errors 784 | if !ok { 785 | t.Fatal("channel closed") 786 | } 787 | } 788 | 789 | func TestAtomizer_Wait(t *testing.T) { 790 | ctx, cancel := _ctx(context.TODO()) 791 | a := &atomizer{ 792 | ctx: ctx, 793 | cancel: cancel, 794 | publisher: event.NewPublisher(ctx), 795 | } 796 | 797 | cancel() 798 | a.Wait() 799 | } 800 | 801 | func TestAtomizer_distribute_closedEchan(t *testing.T) { 802 | ctx, cancel := _ctx(context.TODO()) 803 | a := &atomizer{ 804 | ctx: ctx, 805 | cancel: cancel, 806 | electrons: make(chan instance), 807 | publisher: event.NewPublisher(ctx), 808 | } 809 | close(a.electrons) 810 | 811 | errors := a.Errors(1) 812 | 813 | a.distribute() 814 | 815 | _, ok := <-errors 816 | if !ok { 817 | t.Fatal("channel closed") 818 | } 819 | } 820 | 821 | func TestAtomizer_exec_ERR(t *testing.T) { 822 | ctx, cancel := _ctx(context.TODO()) 823 | a := &atomizer{ 824 | ctx: ctx, 825 | cancel: cancel, 826 | publisher: event.NewPublisher(ctx), 827 | } 828 | 829 | errors := a.Errors(1) 830 | 831 | i := instance{ctx: ctx, cancel: cancel} 832 | 833 | a.exec(i, nil) 834 | 835 | _, ok := <-errors 836 | if !ok { 837 | t.Fatal("channel closed") 838 | } 839 | } 840 | 841 | func unexpHarness(t *testing.T) (context.Context, context.CancelFunc, *atomizer) { 842 | ctx, cancel := _ctx(context.TODO()) 843 | mizer, err := Atomize(ctx) 844 | if err != nil { 845 | t.Fatal(err) 846 | } 847 | 848 | a, ok := mizer.(*atomizer) 849 | if !ok { 850 | t.Fatal("unable to cast atomizer") 851 | } 852 | 853 | return ctx, cancel, a 854 | } 855 | 856 | func TestAtomizer_distribute_unregistered(t *testing.T) { 857 | ctx, cancel, a := unexpHarness(t) 858 | 859 | errors := a.Errors(1) 860 | 861 | i := instance{ 862 | ctx: ctx, 863 | cancel: cancel, 864 | electron: &Electron{AtomID: "nopey.nope"}, 865 | } 866 | 867 | go a.distribute() 868 | go func() { a.electrons <- i }() 869 | 870 | _, ok := <-errors 871 | if !ok { 872 | t.Fatal("channel closed") 873 | } 874 | } 875 | 876 | func TestAtomizer_exec_inst_err(t *testing.T) { 877 | ctx, cancel, a := unexpHarness(t) 878 | 879 | errors := a.Errors(1) 880 | 881 | i := instance{ 882 | ctx: ctx, 883 | cancel: cancel, 884 | electron: noopelectron, 885 | conductor: &noopconductor{}, 886 | } 887 | 888 | go a.exec(i, &panicatom{}) 889 | 890 | _, ok := <-errors 891 | if !ok { 892 | t.Fatal("channel closed") 893 | } 894 | } 895 | 896 | // Validates the instance of the atomizer 897 | func TestAtomizer_Validate(t *testing.T) { 898 | tests := []struct { 899 | key string 900 | value interface{} 901 | err bool 902 | }{ 903 | { 904 | "ValidAtomizerTest", 905 | &atomizer{ 906 | electrons: make(chan instance), 907 | bonded: make(chan instance), 908 | ctx: context.Background(), 909 | cancel: context.CancelFunc(func() { 910 | 911 | }), 912 | publisher: event.NewPublisher(context.Background()), 913 | }, 914 | false, 915 | }, 916 | { 917 | "InvalidAtomizerNilAtomizer", 918 | nil, 919 | true, 920 | }, 921 | } 922 | 923 | for _, test := range tests { 924 | t.Run(test.key, func(t *testing.T) { 925 | ok := validator.Valid(test.value) 926 | if !ok && !test.err { 927 | t.Fatalf("expected success, got error") 928 | } 929 | 930 | if ok && test.err { 931 | t.Fatalf("expected error") 932 | } 933 | }) 934 | } 935 | } 936 | 937 | // ******************************** 938 | // BENCHMARKS 939 | // ******************************** 940 | 941 | func BenchmarkAtomizer_Exec_Single(b *testing.B) { 942 | resetB() 943 | b.Cleanup(func() { 944 | resetB() 945 | }) 946 | 947 | ctx, cancel := _ctx(context.TODO()) 948 | defer cancel() 949 | 950 | conductor, _, err := harness(ctx, -1) 951 | if err != nil { 952 | b.Fatalf("test harness failed [%s]", err.Error()) 953 | } 954 | 955 | // cleanup the benchmark timer to get correct measurements 956 | b.ResetTimer() 957 | 958 | for n := 0; n < b.N; n++ { 959 | e := newElectron(ID(noopatom{}), nil) 960 | 961 | // Send the electron onto the conductor 962 | result, err := conductor.Send(ctx, e) 963 | if err != nil { 964 | b.Fatal(err) 965 | } 966 | 967 | select { 968 | case <-ctx.Done(): 969 | b.Fatal("context closed, test failed") 970 | case result, ok := <-result: 971 | if !ok { 972 | b.Fatal("result channel closed, test failed") 973 | } 974 | 975 | if result.Error != nil { 976 | b.Fatal("Error returned from atom", result.Error) 977 | } 978 | } 979 | } 980 | } 981 | -------------------------------------------------------------------------------- /conductor.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import "context" 9 | 10 | // Conductor is the interface that should be implemented for passing 11 | // electrons to the atomizer that need processing. This should generally be 12 | // registered with the atomizer in an initialization script 13 | type Conductor interface { 14 | 15 | // Receive gets the atoms from the source 16 | // that are available to atomize 17 | Receive(ctx context.Context) <-chan *Electron 18 | 19 | // Complete mark the completion of an electron instance 20 | // with applicable statistics 21 | Complete(ctx context.Context, p *Properties) error 22 | 23 | // Send sends electrons back out through the conductor for 24 | // additional processing 25 | Send(ctx context.Context, electron *Electron) (<-chan *Properties, error) 26 | 27 | // Close cleans up the conductor 28 | Close() 29 | } 30 | -------------------------------------------------------------------------------- /docs/definitions.md: -------------------------------------------------------------------------------- 1 | # Definitions 2 | 3 | ## Atom 4 | 5 | An Atom is a piece of business logic that implements the Process method of the 6 | Atom interface. Atoms are the primary workhorse of the Atomizer. Without their 7 | payload, known as Electrons, Atoms cannot be processed. A bonded Atom (i.e., 8 | instance) is an Atom that has been initialized with a valid Electron. 9 | 10 | ## Conductor 11 | 12 | A Conductor is the “driver” that connects to message queues (or to in-memory 13 | caches or other communication frameworks) for passing processing payloads 14 | (i.e., Electrons) throughout the cluster. This driver implements the Conductor 15 | interface. For example, an Atom that connects to another Atom directly for 16 | processing requests could be a form of Conductor. 17 | 18 | ## Electron 19 | 20 | An Electron is the payload that is retrieved from the message queue or other 21 | communication processor. It contains data in the form of a byte slice (could 22 | be json, protobuf, or gob-encoded, etc...) that is then used to properly 23 | initialize an Atom. Essentially, the Electron is the configuration for the 24 | business logic that is internal to the Atom. 25 | -------------------------------------------------------------------------------- /docs/design-methodologies.md: -------------------------------------------------------------------------------- 1 | # Atomizer Design 2 | 3 | [Atomizer Architecture Diagram][arch] 4 | 5 | ## Design Methodologies 6 | 7 | Atomizer is built with an expectation failure in atoms it executes. This 8 | approach allows the system to adapt to non-critical failures, such as panics 9 | generated by Atoms or Conductors in the system within defined limits. For 10 | example, a system with only one Conductor and which fails has nothing to 11 | relay processing requests and should panic, whereas an instance with multiple 12 | conductors should fail only the singular conductor and continue processing on 13 | the other registered conductors in the system. 14 | 15 | ## Atomizer Framework Failure Modes 16 | 17 | There are two primary failure modes: Critical and Non-Critical. 18 | 19 | * Critical areas of the Atomizer cause a panic and crash the application with 20 | information rich errors in order to alert the user to issues in the Atomizer. 21 | 22 | * Non-Critical areas of the Atomizer capture the panic and push the error 23 | along the event channel so that anything monitoring for events can pick them 24 | up. Rather than crashing the Atomizer, though, the system attempts to 25 | re-initialize the element that failed (self-heal). 26 | 27 | In the event that a message from the message queue is not acknowledged and 28 | the connection between the message queue and a node in the cluster is 29 | terminated, the message queue should attempt to deliver the message to a 30 | different node in the cluster for processing (likely round robin with current 31 | rabbitmq implementations). 32 | 33 | ## Atom Types 34 | 35 | Atoms have several different processing types: Singleton, Spawner 36 | (Wait and Free), and Atomic. 37 | 38 | 1. **Singleton Atoms** are Atoms where there is a single instance of that Atom on 39 | the node at one time. Singleton Atoms will generally be maintenance Atoms for 40 | the distributed system or long running process for the system users. Examples 41 | of singleton Atoms would be the plugin system, which will monitor for new 42 | plugins or live Atom updates from the message queue and deploy those plugins 43 | live in the running environment. Singleton Atoms will also likely maintain an 44 | internal static state. Singleton Atoms can complete their processing and close 45 | down, but there should never be more than one running instance on a node at a 46 | time. 47 | 48 | 2. **Spawner Atoms** come in two forms. 49 | 50 | 1. The Wait spawner initiates 1 -> N instances of different Atoms in the 51 | cluster for processing and awaits their results in order to complete the 52 | algorithm. Monte Carlo *pi* estimation - where the results are returned and 53 | aggregated to calculate the final result - is an example of this kind of 54 | spawner. 55 | 56 | 2. The Free spawner is one in which the Atom initiates one to N additional 57 | Atoms but does not wait for the response. Instead, the results of those 58 | additional Atoms are monitored somewhere else by another process or not 59 | monitored at all. 60 | 61 | 3. **Atomic** is the third form of an Atom and, since it is created for a singular 62 | purpose, is the lowest level form of an Atom. It receives an Electron, 63 | executes the Atom using the Electron information, and returns the result back 64 | to the Conductor. 65 | 66 | ## Atom Failure Modes 67 | 68 | Individual Atom instances have several failure modes that will need to be 69 | configured as part of the Atom’s Electron. 70 | 71 | These failure modes are: Self-Healing / Retry, Log / Fail, and Re-Queue. 72 | 73 | * **Self-Heal / Retry** will push the Electron for the Atom back onto the 74 | Atomizer and attempt to re-run the Atom’s processing method. Self-Heal 75 | will also pass failure information, along with the Electron, for reprocessing 76 | to ensure that the Atomizer can identify systemic failures so that an Atom 77 | that has already been self-healed will not be attempted too many times or 78 | create an infinite loop. Retry configuration would be part of the Electron 79 | information and would tell the Atomizer how many times to heal a failing Atom. 80 | 81 | * **Log / Fail** will log the failed Atom information and return the failure 82 | state back to the message queue as the Electron’s response. 83 | 84 | * **Re-Queue** will follow a similar path as Self-Heal / Retry. However, instead 85 | of re-attempting the processing on the node internally, it will fail the 86 | acknowledgement back to the message queue **or** push the Electron back to the 87 | message queue for distribution to a different node in the cluster for 88 | processing. This process will use the original Conductor the Electron was 89 | received from in the event that there are multiple Conductors on the node. 90 | 91 | 92 | [arch_small]:media/design_small.jpg 93 | [arch]:media/design.jpg 94 | -------------------------------------------------------------------------------- /docs/media/design.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devnw/atomizer/0ab2433cf7738c5978e4490e4c8ed6ce9a7cc368/docs/media/design.jpg -------------------------------------------------------------------------------- /docs/media/design_small.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/devnw/atomizer/0ab2433cf7738c5978e4490e4c8ed6ce9a7cc368/docs/media/design_small.jpg -------------------------------------------------------------------------------- /electron.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "encoding/base64" 10 | "encoding/gob" 11 | "encoding/json" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | func init() { 17 | gob.Register(Electron{}) 18 | } 19 | 20 | // Electron is the base electron that MUST parse from the payload 21 | // from the conductor 22 | type Electron struct { 23 | // SenderID is the unique identifier for the node that sent the 24 | // electron 25 | SenderID string 26 | 27 | // ID is the unique identifier of this electron 28 | ID string 29 | 30 | // AtomID is the identifier of the atom for this electron instance 31 | // this is generally `package.Type`. Use the atomizer.ID() method 32 | // if unsure of the type for an Atom. 33 | AtomID string 34 | 35 | // Timeout is the maximum time duration that should be allowed 36 | // for this instance to process. After the duration is exceeded 37 | // the context should be canceled and the processing released 38 | // and a failure sent back to the conductor 39 | Timeout *time.Duration 40 | 41 | // CopyState lets atomizer know if it should copy the state of the 42 | // original atom registration to the new atom instance when processing 43 | // a newly received electron 44 | // 45 | // NOTE: Copying the state of an Atom as registered requires that ALL 46 | // fields that are to be copied are **EXPORTED** otherwise they are 47 | // skipped 48 | CopyState bool 49 | 50 | // Payload is to be used by the registered atom to properly unmarshal 51 | // the []byte for the actual atom instance. RawMessage is used to 52 | // delay unmarshal of the payload information so the atom can do it 53 | // internally 54 | Payload []byte 55 | } 56 | 57 | // UnmarshalJSON reads in a []byte of JSON data and maps it to the Electron 58 | // struct properly for use throughout Atomizer 59 | func (e *Electron) UnmarshalJSON(data []byte) error { 60 | jsonE := struct { 61 | SenderID string `json:"senderid"` 62 | ID string `json:"id"` 63 | AtomID string `json:"atomid"` 64 | Timeout *time.Duration `json:"timeout,omitempty"` 65 | CopyState bool `json:"copystate,omitempty"` 66 | Payload json.RawMessage `json:"payload,omitempty"` 67 | }{} 68 | 69 | err := json.Unmarshal(data, &jsonE) 70 | if err != nil { 71 | return err 72 | } 73 | 74 | e.SenderID = jsonE.SenderID 75 | e.ID = jsonE.ID 76 | e.AtomID = jsonE.AtomID 77 | e.Timeout = jsonE.Timeout 78 | 79 | if jsonE.Payload != nil { 80 | pay := strings.Trim(string(jsonE.Payload), "\"") 81 | e.Payload, err = base64.StdEncoding.DecodeString(pay) 82 | if err != nil { 83 | e.Payload = jsonE.Payload 84 | } 85 | } 86 | 87 | return nil 88 | } 89 | 90 | // MarshalJSON implements the custom json marshaler for electron 91 | func (e *Electron) MarshalJSON() ([]byte, error) { 92 | return json.Marshal(&struct { 93 | SenderID string `json:"senderid"` 94 | ID string `json:"id"` 95 | AtomID string `json:"atomid"` 96 | Timeout *time.Duration `json:"timeout,omitempty"` 97 | CopyState bool `json:"copystate,omitempty"` 98 | Payload json.RawMessage `json:"payload,omitempty"` 99 | }{ 100 | SenderID: e.SenderID, 101 | ID: e.ID, 102 | AtomID: e.AtomID, 103 | Timeout: e.Timeout, 104 | Payload: json.RawMessage(e.Payload), 105 | }) 106 | } 107 | 108 | // Validate ensures that the electron information is intact for proper 109 | // execution 110 | func (e *Electron) Validate() (valid bool) { 111 | if e != nil && 112 | e.SenderID != "" && 113 | e.ID != "" && 114 | e.AtomID != "" { 115 | valid = true 116 | } 117 | 118 | return valid 119 | } 120 | -------------------------------------------------------------------------------- /electron_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/google/go-cmp/cmp" 10 | "go.devnw.com/validator" 11 | ) 12 | 13 | var pay = `{"test":"test"}` 14 | var pay64Encoded = `eyJ0ZXN0IjoidGVzdCJ9` 15 | 16 | var nonb64 = &Electron{ 17 | SenderID: "empty", 18 | ID: "empty", 19 | AtomID: "empty", 20 | Payload: []byte(pay), 21 | } 22 | 23 | func TestElectron_MarshalJSON(t *testing.T) { 24 | tests := []struct { 25 | name string 26 | e *Electron 27 | expected string 28 | err bool 29 | }{ 30 | { 31 | "valid electron", 32 | noopelectron, 33 | `{"senderid":"empty","id":"empty","atomid":"empty"}`, 34 | false, 35 | }, 36 | { 37 | "valid electron w/ payload", 38 | nonb64, 39 | fmt.Sprintf(`{"senderid":"empty","id":"empty","atomid":"empty","payload":%s}`, pay), 40 | false, 41 | }, 42 | } 43 | 44 | for _, test := range tests { 45 | t.Run(test.name, func(t *testing.T) { 46 | res, err := json.Marshal(test.e) 47 | if err != nil && !test.err { 48 | t.Fatalf("expected success, got error | %s", err.Error()) 49 | } 50 | 51 | if err == nil && test.err { 52 | t.Fatal("expected error") 53 | } 54 | 55 | if strings.Compare(string(res), test.expected) != 0 { 56 | t.Fatalf( 57 | "mismatch: e[%s] != r[%s]", 58 | test.expected, 59 | string(res), 60 | ) 61 | } 62 | }) 63 | } 64 | } 65 | 66 | func TestElectron_UnmarshalJSON(t *testing.T) { 67 | tests := []struct { 68 | name string 69 | expected *Electron 70 | json string 71 | err bool 72 | }{ 73 | { 74 | "valid electron", 75 | noopelectron, 76 | `{"senderid":"empty","id":"empty","atomid":"empty"}`, 77 | false, 78 | }, 79 | { 80 | "valid electron / non-base64 payload", 81 | nonb64, 82 | `{"senderid":"empty","id":"empty","atomid":"empty","payload":{"test":"test"}}`, 83 | false, 84 | }, 85 | { 86 | "valid electron / base64 payload", 87 | nonb64, 88 | fmt.Sprintf(`{"senderid":"empty","id":"empty","atomid":"empty","payload":%q}`, pay64Encoded), 89 | false, 90 | }, 91 | { 92 | "invalid json blob", 93 | &Electron{}, 94 | `{"empty"}`, 95 | true, 96 | }, 97 | } 98 | 99 | for _, test := range tests { 100 | t.Run(test.name, func(t *testing.T) { 101 | e := &Electron{} 102 | err := json.Unmarshal([]byte(test.json), &e) 103 | 104 | if err != nil && !test.err { 105 | t.Fatalf("expected success, got error | %s", err.Error()) 106 | } 107 | 108 | if err == nil && test.err { 109 | t.Fatal("expected error") 110 | } 111 | 112 | diff := cmp.Diff(test.expected, e) 113 | if diff != "" { 114 | t.Fatalf( 115 | "expected equality %s", 116 | diff, 117 | ) 118 | } 119 | }) 120 | } 121 | } 122 | 123 | func TestElectron_Validate(t *testing.T) { 124 | tests := []struct { 125 | name string 126 | e *Electron 127 | valid bool 128 | }{ 129 | { 130 | "valid electron", 131 | noopelectron, 132 | true, 133 | }, 134 | { 135 | "invalid electron", 136 | &Electron{}, 137 | false, 138 | }, 139 | { 140 | "invalid electron / only sender", 141 | &Electron{SenderID: "test"}, 142 | false, 143 | }, 144 | { 145 | "invalid electron / only atom", 146 | &Electron{AtomID: "test"}, 147 | false, 148 | }, 149 | { 150 | "invalid electron / only ID", 151 | &Electron{ID: "test"}, 152 | false, 153 | }, 154 | { 155 | "invalid electron / sender & atom", 156 | &Electron{SenderID: "test", AtomID: "test"}, 157 | false, 158 | }, 159 | { 160 | "invalid electron / ID & sender", 161 | &Electron{ID: "test", SenderID: "test"}, 162 | false, 163 | }, 164 | { 165 | "invalid electron / ID & atom", 166 | &Electron{ID: "test", AtomID: "test"}, 167 | false, 168 | }, 169 | } 170 | 171 | for _, test := range tests { 172 | t.Run(test.name, func(t *testing.T) { 173 | if !validator.Valid(test.e) == test.valid { 174 | t.Fatalf("expected valid = %v", test.valid) 175 | } 176 | }) 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /error.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "encoding/gob" 10 | "fmt" 11 | "strings" 12 | ) 13 | 14 | func init() { 15 | gob.Register(&Error{}) 16 | } 17 | 18 | type wrappedErr interface { 19 | Unwrap() error 20 | } 21 | 22 | func simple(msg string, internal error) error { 23 | return &Error{ 24 | Event: &Event{ 25 | Message: msg, 26 | }, 27 | Internal: internal, 28 | } 29 | } 30 | 31 | // ptoe takes a result of a recover and coverts it 32 | // to a string 33 | func ptoe(r interface{}) error { 34 | return &Error{ 35 | Event: makeEvent(ptos(r)), 36 | } 37 | } 38 | 39 | // ptos takes a result of a recover and coverts it 40 | // to a string 41 | func ptos(r interface{}) string { 42 | return fmt.Sprintf("%v", r) 43 | } 44 | 45 | // Error is an error type which provides specific 46 | // atomizer information as part of an error 47 | type Error struct { 48 | 49 | // Event is the event that took place to create 50 | // the error and contains metadata relevant to the error 51 | Event *Event `json:"event"` 52 | 53 | // Internal is the internal error 54 | Internal error `json:"internal"` 55 | } 56 | 57 | func (e *Error) Error() string { 58 | return e.String() 59 | } 60 | 61 | func (e *Error) String() string { 62 | var fields []string 63 | 64 | fields = append(fields, "atomizer error") 65 | 66 | msg := e.Event.Event() 67 | if msg != "" { 68 | fields = append(fields, msg) 69 | } 70 | 71 | if e.Internal != nil { 72 | fields = append( 73 | fields, 74 | "| internal: ("+e.Internal.Error()+")", 75 | ) 76 | } 77 | 78 | return strings.Join(fields, " ") 79 | } 80 | 81 | // Unwrap unwraps the error to the deepest error and returns that one 82 | func (e *Error) Unwrap() (err error) { 83 | err = e.Internal 84 | 85 | // Determine if the internal error implements 86 | // the wrappedErr interface then continue unwrapping 87 | // if it does 88 | if internal, ok := err.(wrappedErr); ok { 89 | // Recursive unwrap to get the lowest error 90 | err = internal.Unwrap() 91 | } 92 | 93 | return err 94 | } 95 | 96 | // Validate determines if this is a properly built error 97 | func (e *Error) Validate() bool { 98 | return e.Event.Validate() 99 | } 100 | -------------------------------------------------------------------------------- /error_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pkg/errors" 7 | "go.devnw.com/validator" 8 | ) 9 | 10 | func TestError_Error(t *testing.T) { 11 | prefix := "atomizer error" 12 | 13 | tests := []struct { 14 | name string 15 | e error 16 | expected string 17 | }{ 18 | { 19 | "error w/message test", 20 | &Error{ 21 | Event: &Event{ 22 | Message: "test", 23 | }, 24 | }, 25 | prefix + " test", 26 | }, 27 | { 28 | "error w/empty message test", 29 | &Error{ 30 | Event: &Event{}, 31 | }, 32 | prefix, 33 | }, 34 | { 35 | "error w/inner error test", 36 | &Error{ 37 | Internal: &Error{ 38 | Event: &Event{ 39 | Message: "test", 40 | }, 41 | }, 42 | }, 43 | prefix + " | internal: (atomizer error test)", 44 | }, 45 | { 46 | "error w/multiple inner error test", 47 | &Error{ 48 | Internal: &Error{ 49 | Event: &Event{ 50 | Message: "test", 51 | }, 52 | Internal: &Error{ 53 | Event: &Event{ 54 | Message: "test 2", 55 | }, 56 | }, 57 | }, 58 | }, 59 | prefix + " | internal: (atomizer error test" + 60 | " | internal: (atomizer error test 2))", 61 | }, 62 | { 63 | "simple error test", 64 | simple("test", nil), 65 | prefix + " test", 66 | }, 67 | { 68 | "simple error w/inner test", 69 | simple( 70 | "test", 71 | simple("inner", nil), 72 | ), 73 | prefix + " test | internal: (atomizer error inner)", 74 | }, 75 | } 76 | 77 | for _, test := range tests { 78 | t.Run(test.name, func(t *testing.T) { 79 | result := test.e.Error() 80 | if result != test.expected { 81 | t.Fatalf( 82 | "expected [%s] got [%s]", 83 | test.expected, 84 | result, 85 | ) 86 | } 87 | }) 88 | } 89 | } 90 | 91 | func TestError_Unwrap(t *testing.T) { 92 | e := errors.New("wrapped error") 93 | 94 | aerr := &Error{ 95 | Internal: &Error{ 96 | Internal: e, 97 | }, 98 | } 99 | 100 | if aerr.Unwrap() != e { 101 | t.Fatalf( 102 | "expected [%v] got [%v]", 103 | e, 104 | aerr.Unwrap(), 105 | ) 106 | } 107 | } 108 | 109 | func TestError_Unwrap_Fail(t *testing.T) { 110 | e := errors.New("wrapped error") 111 | 112 | aerr := &Error{ 113 | Internal: e, 114 | } 115 | 116 | if aerr.Unwrap() != e { 117 | t.Fatalf( 118 | "expected [%v] got [%v]", 119 | e, 120 | aerr.Unwrap(), 121 | ) 122 | } 123 | } 124 | 125 | func TestError_Validate(t *testing.T) { 126 | tests := []struct { 127 | name string 128 | e error 129 | valid bool 130 | }{ 131 | { 132 | "error w/valid event", 133 | &Error{ 134 | Event: &Event{ 135 | Message: "test", 136 | }, 137 | }, 138 | true, 139 | }, 140 | { 141 | "error w/invalid event", 142 | &Error{ 143 | Event: &Event{}, 144 | }, 145 | false, 146 | }, 147 | } 148 | 149 | for _, test := range tests { 150 | t.Run(test.name, func(t *testing.T) { 151 | v := validator.Valid(test.e) 152 | if test.valid != v { 153 | t.Fatalf( 154 | "expected [%v] got [%v]", 155 | test.valid, 156 | v, 157 | ) 158 | } 159 | }) 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /event.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "encoding/gob" 10 | "strings" 11 | ) 12 | 13 | func init() { 14 | gob.Register(&Event{}) 15 | } 16 | 17 | // Event indicates an atomizer event has taken 18 | // place that is not categorized as an error 19 | // Event implements the stringer interface but 20 | // does NOT implement the error interface 21 | type Event struct { 22 | // Message from atomizer about this error 23 | Message string `json:"message"` 24 | 25 | // ElectronID is the associated electron instance 26 | // where the error occurred. Empty ElectronID indicates 27 | // the error was not part of a running electron instance. 28 | ElectronID string `json:"electronID"` 29 | 30 | // AtomID is the atom which was processing when 31 | // the error occurred. Empty AtomID indicates 32 | // the error was not part of a running atom. 33 | AtomID string `json:"atomID"` 34 | 35 | // ConductorID is the conductor which was being 36 | // used for receiving instructions 37 | ConductorID string `json:"conductorID"` 38 | } 39 | 40 | func (e *Event) Event() string { 41 | return e.String() 42 | } 43 | 44 | func (e *Event) String() string { 45 | if e == nil { 46 | return "" 47 | } 48 | 49 | var joins []string 50 | 51 | ids := e.ids() 52 | if ids != "" { 53 | joins = append(joins, "["+ids+"]") 54 | } 55 | 56 | // Add the message to part of the error 57 | if e.Message != "" { 58 | joins = append(joins, e.Message) 59 | } 60 | 61 | return strings.Join(joins, " ") 62 | } 63 | 64 | // ids returns the electron and atom ids as a combination string 65 | func (e *Event) ids() string { 66 | var ids []string 67 | 68 | // Include the conductor id if it is part of the event 69 | if e.ConductorID != "" { 70 | ids = append(ids, "cid:"+e.ConductorID) 71 | } 72 | 73 | // Include the atom id if it is part of the event 74 | if e.AtomID != "" { 75 | ids = append(ids, "aid:"+e.AtomID) 76 | } 77 | 78 | // Include the electron id if it is part of the event 79 | if e.ElectronID != "" { 80 | ids = append(ids, "eid:"+e.ElectronID) 81 | } 82 | 83 | return strings.Join(ids, " | ") 84 | } 85 | 86 | // makeEvent creates a base event with only a message and 87 | // a time 88 | func makeEvent(msg string) *Event { 89 | return &Event{ 90 | Message: msg, 91 | } 92 | } 93 | 94 | // Validate determines if the event is valid 95 | func (e *Event) Validate() bool { 96 | return e.Message != "" 97 | } 98 | -------------------------------------------------------------------------------- /event_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestEvent_String(t *testing.T) { 8 | tests := []struct { 9 | name string 10 | e *Event 11 | expected string 12 | }{ 13 | { 14 | "message test", 15 | &Event{ 16 | Message: "test", 17 | }, 18 | "test", 19 | }, 20 | { 21 | "makeEvent message test", 22 | makeEvent("test"), 23 | "test", 24 | }, 25 | { 26 | "message w/a test", 27 | &Event{ 28 | Message: "test", 29 | AtomID: "10", 30 | }, 31 | "[aid:10] test", 32 | }, 33 | { 34 | "message w/c test", 35 | &Event{ 36 | Message: "test", 37 | ConductorID: "10", 38 | }, 39 | "[cid:10] test", 40 | }, 41 | { 42 | "message w/ea test", 43 | &Event{ 44 | Message: "test", 45 | ElectronID: "10", 46 | AtomID: "11", 47 | }, 48 | "[aid:11 | eid:10] test", 49 | }, 50 | { 51 | "message w/eac test", 52 | &Event{ 53 | Message: "test", 54 | ElectronID: "10", 55 | AtomID: "11", 56 | ConductorID: "12", 57 | }, 58 | "[cid:12 | aid:11 | eid:10] test", 59 | }, 60 | { 61 | "eac test", 62 | &Event{ 63 | ElectronID: "10", 64 | AtomID: "11", 65 | ConductorID: "12", 66 | }, 67 | "[cid:12 | aid:11 | eid:10]", 68 | }, 69 | } 70 | 71 | for _, test := range tests { 72 | t.Run(test.name, func(t *testing.T) { 73 | result := test.e.String() 74 | if result != test.expected { 75 | t.Fatalf( 76 | "expected [%s] got [%s]", 77 | test.expected, 78 | result, 79 | ) 80 | } 81 | }) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "_1password-shell-plugins": { 4 | "inputs": { 5 | "flake-utils": "flake-utils", 6 | "nixpkgs": "nixpkgs" 7 | }, 8 | "locked": { 9 | "lastModified": 1727269329, 10 | "narHash": "sha256-GUqMvwmoVVCHq95WpZa2bIQ6sALL9qlld7HTxPWRcl0=", 11 | "owner": "1Password", 12 | "repo": "shell-plugins", 13 | "rev": "36e4bf4d5b177dd059cafdafaed34bd3264e1e2e", 14 | "type": "github" 15 | }, 16 | "original": { 17 | "owner": "1Password", 18 | "repo": "shell-plugins", 19 | "type": "github" 20 | } 21 | }, 22 | "flake-utils": { 23 | "inputs": { 24 | "systems": "systems" 25 | }, 26 | "locked": { 27 | "lastModified": 1710146030, 28 | "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", 29 | "owner": "numtide", 30 | "repo": "flake-utils", 31 | "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", 32 | "type": "github" 33 | }, 34 | "original": { 35 | "owner": "numtide", 36 | "repo": "flake-utils", 37 | "type": "github" 38 | } 39 | }, 40 | "flake-utils_2": { 41 | "inputs": { 42 | "systems": "systems_2" 43 | }, 44 | "locked": { 45 | "lastModified": 1726560853, 46 | "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", 47 | "owner": "numtide", 48 | "repo": "flake-utils", 49 | "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", 50 | "type": "github" 51 | }, 52 | "original": { 53 | "id": "flake-utils", 54 | "type": "indirect" 55 | } 56 | }, 57 | "flake-utils_3": { 58 | "inputs": { 59 | "systems": "systems_3" 60 | }, 61 | "locked": { 62 | "lastModified": 1694529238, 63 | "narHash": "sha256-zsNZZGTGnMOf9YpHKJqMSsa0dXbfmxeoJ7xHlrt+xmY=", 64 | "owner": "numtide", 65 | "repo": "flake-utils", 66 | "rev": "ff7b65b44d01cf9ba6a71320833626af21126384", 67 | "type": "github" 68 | }, 69 | "original": { 70 | "owner": "numtide", 71 | "repo": "flake-utils", 72 | "type": "github" 73 | } 74 | }, 75 | "gomod2nix": { 76 | "inputs": { 77 | "flake-utils": "flake-utils_3", 78 | "nixpkgs": [ 79 | "nixpkgs" 80 | ] 81 | }, 82 | "locked": { 83 | "lastModified": 1729448365, 84 | "narHash": "sha256-oquZeWTYWTr5IxfwEzgsxjtD8SSFZYLdO9DaQb70vNU=", 85 | "owner": "tweag", 86 | "repo": "gomod2nix", 87 | "rev": "5d387097aa716f35dd99d848dc26d8d5b62a104c", 88 | "type": "github" 89 | }, 90 | "original": { 91 | "owner": "tweag", 92 | "repo": "gomod2nix", 93 | "type": "github" 94 | } 95 | }, 96 | "nixpkgs": { 97 | "locked": { 98 | "lastModified": 1716358718, 99 | "narHash": "sha256-NQbegJb2ZZnAqp2EJhWwTf6DrZXSpA6xZCEq+RGV1r0=", 100 | "owner": "NixOS", 101 | "repo": "nixpkgs", 102 | "rev": "3f316d2a50699a78afe5e77ca486ad553169061e", 103 | "type": "github" 104 | }, 105 | "original": { 106 | "owner": "NixOS", 107 | "ref": "nixpkgs-unstable", 108 | "repo": "nixpkgs", 109 | "type": "github" 110 | } 111 | }, 112 | "nixpkgs_2": { 113 | "locked": { 114 | "lastModified": 1727802920, 115 | "narHash": "sha256-HP89HZOT0ReIbI7IJZJQoJgxvB2Tn28V6XS3MNKnfLs=", 116 | "owner": "NixOS", 117 | "repo": "nixpkgs", 118 | "rev": "27e30d177e57d912d614c88c622dcfdb2e6e6515", 119 | "type": "github" 120 | }, 121 | "original": { 122 | "owner": "NixOS", 123 | "ref": "nixos-unstable", 124 | "repo": "nixpkgs", 125 | "type": "github" 126 | } 127 | }, 128 | "root": { 129 | "inputs": { 130 | "_1password-shell-plugins": "_1password-shell-plugins", 131 | "flake-utils": "flake-utils_2", 132 | "gomod2nix": "gomod2nix", 133 | "nixpkgs": "nixpkgs_2", 134 | "utils": "utils" 135 | } 136 | }, 137 | "systems": { 138 | "locked": { 139 | "lastModified": 1681028828, 140 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 141 | "owner": "nix-systems", 142 | "repo": "default", 143 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 144 | "type": "github" 145 | }, 146 | "original": { 147 | "owner": "nix-systems", 148 | "repo": "default", 149 | "type": "github" 150 | } 151 | }, 152 | "systems_2": { 153 | "locked": { 154 | "lastModified": 1681028828, 155 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 156 | "owner": "nix-systems", 157 | "repo": "default", 158 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 159 | "type": "github" 160 | }, 161 | "original": { 162 | "owner": "nix-systems", 163 | "repo": "default", 164 | "type": "github" 165 | } 166 | }, 167 | "systems_3": { 168 | "locked": { 169 | "lastModified": 1681028828, 170 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 171 | "owner": "nix-systems", 172 | "repo": "default", 173 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 174 | "type": "github" 175 | }, 176 | "original": { 177 | "owner": "nix-systems", 178 | "repo": "default", 179 | "type": "github" 180 | } 181 | }, 182 | "systems_4": { 183 | "locked": { 184 | "lastModified": 1681028828, 185 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 186 | "owner": "nix-systems", 187 | "repo": "default", 188 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 189 | "type": "github" 190 | }, 191 | "original": { 192 | "owner": "nix-systems", 193 | "repo": "default", 194 | "type": "github" 195 | } 196 | }, 197 | "utils": { 198 | "inputs": { 199 | "systems": "systems_4" 200 | }, 201 | "locked": { 202 | "lastModified": 1726560853, 203 | "narHash": "sha256-X6rJYSESBVr3hBoH0WbKE5KvhPU5bloyZ2L4K60/fPQ=", 204 | "owner": "numtide", 205 | "repo": "flake-utils", 206 | "rev": "c1dfcf08411b08f6b8615f7d8971a2bfa81d5e8a", 207 | "type": "github" 208 | }, 209 | "original": { 210 | "owner": "numtide", 211 | "repo": "flake-utils", 212 | "type": "github" 213 | } 214 | } 215 | }, 216 | "root": "root", 217 | "version": 7 218 | } 219 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "development flake"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; 6 | utils.url = "github:numtide/flake-utils"; 7 | _1password-shell-plugins.url = "github:1Password/shell-plugins"; 8 | gomod2nix = { 9 | url = "github:tweag/gomod2nix"; 10 | inputs.nixpkgs.follows = "nixpkgs"; 11 | inputs.utils.follows = "utils"; 12 | }; 13 | }; 14 | 15 | outputs = 16 | { 17 | self, 18 | nixpkgs, 19 | flake-utils, 20 | gomod2nix, 21 | ... 22 | }: 23 | flake-utils.lib.eachDefaultSystem ( 24 | system: 25 | let 26 | pkgs = import nixpkgs { 27 | config = { 28 | allowUnfree = true; 29 | }; 30 | system = system; 31 | overlays = [ 32 | gomod2nix.overlays.default 33 | (self: super: { 34 | go = super.go_1_23; 35 | python = super.python3.withPackages ( 36 | subpkgs: with subpkgs; [ 37 | openapi-spec-validator 38 | detect-secrets 39 | requests 40 | python-dotenv 41 | ] 42 | ); 43 | }) 44 | ]; 45 | }; 46 | 47 | pkglist = with pkgs; [ 48 | _1password 49 | gibberish-detector 50 | addlicense 51 | shfmt 52 | git 53 | pre-commit 54 | shellcheck 55 | automake 56 | act 57 | gcc 58 | 59 | python 60 | 61 | go 62 | gopls 63 | gotools 64 | go-tools 65 | gomod2nix.packages.${system}.default 66 | sqlite-interactive 67 | 68 | delve 69 | golangci-lint 70 | goreleaser 71 | go-licenses 72 | ]; 73 | in 74 | { 75 | packages.default = pkgs.buildGoApplication { 76 | pname = "{{module_name}}"; 77 | version = "0.1"; 78 | 79 | pwd = ./.; 80 | src = ./.; 81 | modules = ./gomod2nix.toml; 82 | buildInputs = pkglist; 83 | 84 | buildPhase = '' 85 | make build 86 | ''; 87 | installPhase = '' 88 | make install 89 | ''; 90 | }; 91 | 92 | devShells.default = pkgs.mkShell { buildInputs = pkglist; }; 93 | } 94 | ); 95 | } 96 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module go.atomizer.io/engine 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/Pallinder/go-randomdata v1.2.0 7 | github.com/davecgh/go-spew v1.1.1 8 | github.com/google/go-cmp v0.6.0 9 | github.com/google/uuid v1.6.0 10 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 11 | github.com/pkg/errors v0.9.1 12 | go.devnw.com/alog v1.1.1 13 | go.devnw.com/event v1.0.3 14 | go.devnw.com/validator v1.0.8 15 | golang.org/x/text v0.3.8 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= 2 | github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 6 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 7 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 8 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 9 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 10 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= 11 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= 12 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 13 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 14 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 15 | go.devnw.com/alog v1.1.1 h1:u+85qyAwvcm0t46brAI8/DNW1FhW+XV+2Xk9MCs621I= 16 | go.devnw.com/alog v1.1.1/go.mod h1:+C3sYPkWZKet8XEvZ7DCcmEa30QQ4BRl67UcGLywOXc= 17 | go.devnw.com/event v1.0.3 h1:8vcAFikzc39vvDEmIhdU4NfH/h80v6iyQu8BnlwEH3A= 18 | go.devnw.com/event v1.0.3/go.mod h1:wc9P8ENcNDAxrIaOdqfPKgNbDwYcuvgYK60KgM61o3s= 19 | go.devnw.com/validator v1.0.8 h1:IUuu8o7WKZPf5WsiUWvdU7qwBZvWwKezpZQnXn2gX+k= 20 | go.devnw.com/validator v1.0.8/go.mod h1:5kfqytQCBGTupXx1ZoIfV97qvKtsAi6lt2HfmvjWa6A= 21 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 22 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 23 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 24 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 25 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 26 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 27 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 28 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 29 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 30 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 31 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 32 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 33 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 34 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 35 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 36 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 37 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 38 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 39 | golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= 40 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 41 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 42 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 43 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 44 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 45 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 46 | -------------------------------------------------------------------------------- /helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | // _ctx returns a valid context with cancel even if it the 16 | // supplied context is initially nil. If the supplied context 17 | // is nil it uses the background context 18 | func _ctx(c context.Context) (context.Context, context.CancelFunc) { 19 | if c == nil { 20 | c = context.Background() 21 | } 22 | 23 | return context.WithCancel(c) 24 | } 25 | 26 | // _ctxT returns a context with a timeout that is passed in as a time.Duration 27 | func _ctxT( 28 | c context.Context, 29 | duration *time.Duration, 30 | ) (context.Context, context.CancelFunc) { 31 | if c == nil { 32 | c = context.Background() 33 | } 34 | 35 | if duration == nil { 36 | return _ctx(c) 37 | } 38 | 39 | return context.WithTimeout(c, *duration) 40 | } 41 | 42 | // ID returns the registration id for the passed in object type 43 | func ID(v interface{}) string { 44 | return strings.Trim(fmt.Sprintf("%T", v), "*") 45 | } 46 | -------------------------------------------------------------------------------- /instance.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "context" 10 | "time" 11 | 12 | "go.devnw.com/validator" 13 | ) 14 | 15 | type instance struct { 16 | electron *Electron 17 | conductor Conductor 18 | atom Atom 19 | properties *Properties 20 | ctx context.Context 21 | cancel context.CancelFunc 22 | 23 | // TODO: add an actions channel here that the monitor can keep 24 | // an eye on for this bonded electron/atom combo 25 | } 26 | 27 | // bond bonds an instance of an electron with an instance of the 28 | // corresponding atom in the atomizer registrations such that 29 | // the execute method of the instance can properly exercise the 30 | // Process method of the interface 31 | func (i *instance) bond(atom Atom) (err error) { 32 | if err = validator.Assert( 33 | i.electron, 34 | i.conductor, 35 | atom, 36 | ); err != nil { 37 | return &Error{ 38 | Event: &Event{ 39 | Message: "error while bonding atom instance", 40 | AtomID: ID(atom), 41 | }, 42 | Internal: err, 43 | } 44 | } 45 | 46 | // register the atom internally because 47 | // the instance is valid 48 | i.atom = atom 49 | 50 | return nil 51 | } 52 | 53 | // complete marks the completion of execution and pushes 54 | // the results to the conductor 55 | func (i *instance) complete() error { 56 | // Set the end time and status in the properties 57 | i.properties.End = time.Now() 58 | 59 | if !validator.Valid(i.conductor) { 60 | return &Error{ 61 | Event: &Event{ 62 | Message: "conductor validation failed", 63 | AtomID: ID(i.atom), 64 | ElectronID: i.electron.ID, 65 | }, 66 | } 67 | } 68 | 69 | // Push the completed instance properties to the conductor 70 | return i.conductor.Complete(i.ctx, i.properties) 71 | } 72 | 73 | // execute runs the process method on the bonded atom / electron pair 74 | func (i *instance) execute(ctx context.Context) (err error) { 75 | defer func() { 76 | if r := recover(); r != nil { 77 | err = &Error{ 78 | Event: &Event{ 79 | Message: "panic in atomizer", 80 | AtomID: ID(i.atom), 81 | ElectronID: i.electron.ID, 82 | }, 83 | Internal: ptoe(r), 84 | } 85 | 86 | return 87 | } 88 | 89 | // ensure that when this method exits the completion 90 | // of this instance takes place and is pushed to the 91 | // conductor 92 | err = i.complete() 93 | }() 94 | 95 | // ensure the instance is valid before attempting 96 | // to execute processing 97 | if !validator.Valid(i) { 98 | return &Error{ 99 | Event: &Event{ 100 | Message: "instance validation failed", 101 | AtomID: ID(i.atom), 102 | }, 103 | } 104 | } 105 | 106 | // Establish internal context 107 | i.ctx, i.cancel = _ctxT(ctx, i.electron.Timeout) 108 | 109 | i.properties = &Properties{ 110 | ElectronID: i.electron.ID, 111 | AtomID: ID(i.atom), 112 | Start: time.Now(), 113 | } 114 | 115 | // TODO: Setup with a heartbeat for monitoring processing of the 116 | // bonded atom stream in from the process method 117 | 118 | // Execute the process method of the atom 119 | i.properties.Result, i.properties.Error = i.atom.Process( 120 | i.ctx, i.conductor, i.electron) 121 | 122 | // TODO: The processing has finished for this bonded atom and the 123 | // results need to be calculated and the properties sent back to the 124 | // conductor 125 | 126 | // TODO: Ensure a is the proper thing to do here?? I think it needs 127 | // to close a out at the conductor rather than here... unless the 128 | // conductor overrode the call back 129 | 130 | // TODO: Execute the callback with the notification here? 131 | // TODO: determine if this is the correct location or if this is \ 132 | // something that should be handled purely by the conductor 133 | 134 | // TODO: Handle this properly 135 | // if inst.electron.Resp != nil { 136 | // // Drop the return for this electron onto the channel 137 | // // to be sent back to the requester 138 | // inst.electron.Resp <- inst.properties 139 | // } 140 | 141 | return nil 142 | } 143 | 144 | // Validate ensures that the instance has the correct 145 | // non-nil values internally so that it functions properly 146 | func (i *instance) Validate() (valid bool) { 147 | if i != nil { 148 | if validator.Valid( 149 | i.electron, 150 | i.conductor, 151 | i.atom) { 152 | valid = true 153 | } 154 | } 155 | 156 | return valid 157 | } 158 | 159 | func NewTime(ctx context.Context) <-chan time.Time { 160 | tchan := make(chan time.Time) 161 | 162 | go func(tchan chan<- time.Time) { 163 | defer close(tchan) 164 | 165 | for { 166 | select { 167 | case <-ctx.Done(): 168 | return 169 | case tchan <- time.Now(): 170 | } 171 | } 172 | }(tchan) 173 | 174 | return tchan 175 | } 176 | -------------------------------------------------------------------------------- /instance_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | func Test_instance_bond(t *testing.T) { 9 | ctx, cancel := _ctx(context.TODO()) 10 | 11 | tests := []struct { 12 | name string 13 | inst instance 14 | atom Atom 15 | err bool 16 | }{ 17 | { 18 | "valid instance", 19 | instance{ 20 | electron: noopelectron, 21 | conductor: &noopconductor{}, 22 | properties: &Properties{}, 23 | ctx: ctx, 24 | cancel: cancel, 25 | }, 26 | &noopatom{}, 27 | false, 28 | }, 29 | { 30 | "invalid instance / missing electron", 31 | instance{ 32 | conductor: &noopconductor{}, 33 | properties: &Properties{}, 34 | ctx: ctx, 35 | cancel: cancel, 36 | }, 37 | &noopatom{}, 38 | true, 39 | }, 40 | { 41 | "invalid instance / missing conductor", 42 | instance{ 43 | electron: noopelectron, 44 | properties: &Properties{}, 45 | ctx: ctx, 46 | cancel: cancel, 47 | }, 48 | &noopatom{}, 49 | true, 50 | }, 51 | { 52 | "invalid instance / nil atom", 53 | instance{ 54 | electron: noopelectron, 55 | conductor: &noopconductor{}, 56 | properties: &Properties{}, 57 | ctx: ctx, 58 | cancel: cancel, 59 | }, 60 | nil, 61 | true, 62 | }, 63 | } 64 | 65 | for _, test := range tests { 66 | t.Run(test.name, func(t *testing.T) { 67 | err := test.inst.bond(test.atom) 68 | if err != nil && !test.err { 69 | t.Fatalf( 70 | "expected success, got error %s", 71 | err, 72 | ) 73 | } 74 | }) 75 | } 76 | } 77 | 78 | func Test_instance_complete(t *testing.T) { 79 | ctx, cancel := _ctx(context.TODO()) 80 | 81 | tests := []struct { 82 | name string 83 | inst instance 84 | atom Atom 85 | err bool 86 | }{ 87 | { 88 | "valid instance", 89 | instance{ 90 | electron: noopelectron, 91 | conductor: &noopconductor{}, 92 | properties: &Properties{}, 93 | ctx: ctx, 94 | cancel: cancel, 95 | }, 96 | &noopatom{}, 97 | false, 98 | }, 99 | } 100 | 101 | for _, test := range tests { 102 | t.Run(test.name, func(t *testing.T) { 103 | err := test.inst.bond(test.atom) 104 | if err != nil && !test.err { 105 | t.Fatalf( 106 | "expected success, got error %s", 107 | err, 108 | ) 109 | } 110 | }) 111 | } 112 | } 113 | 114 | func Test_instance_execute(t *testing.T) { 115 | ctx, _ := _ctx(context.TODO()) 116 | 117 | tests := []struct { 118 | name string 119 | inst instance 120 | err bool 121 | }{ 122 | { 123 | "valid instance", 124 | instance{ 125 | electron: noopelectron, 126 | conductor: &noopconductor{}, 127 | properties: &Properties{}, 128 | atom: &noopatom{}, 129 | }, 130 | false, 131 | }, 132 | { 133 | "invalid instance - no electron", 134 | instance{ 135 | conductor: &noopconductor{}, 136 | properties: &Properties{}, 137 | atom: &noopatom{}, 138 | }, 139 | true, 140 | }, 141 | { 142 | "invalid instance - no conductor", 143 | instance{ 144 | electron: noopelectron, 145 | properties: &Properties{}, 146 | atom: &noopatom{}, 147 | }, 148 | true, 149 | }, 150 | { 151 | "invalid instance - no atom", 152 | instance{ 153 | electron: noopelectron, 154 | conductor: &noopconductor{}, 155 | properties: &Properties{}, 156 | }, 157 | true, 158 | }, 159 | { 160 | "invalid instance - panic atom", 161 | instance{ 162 | electron: noopelectron, 163 | conductor: &noopconductor{}, 164 | properties: &Properties{}, 165 | atom: &panicatom{}, 166 | }, 167 | true, 168 | }, 169 | } 170 | 171 | for _, test := range tests { 172 | t.Run(test.name, func(t *testing.T) { 173 | err := test.inst.execute(ctx) 174 | if err != nil && !test.err { 175 | t.Fatalf( 176 | "expected success, got error %s", 177 | err, 178 | ) 179 | } 180 | }) 181 | } 182 | } 183 | 184 | func Test_instance_Validate(t *testing.T) { 185 | tests := []struct { 186 | name string 187 | inst instance 188 | valid bool 189 | }{ 190 | { 191 | "valid instance", 192 | instance{ 193 | electron: noopelectron, 194 | conductor: &noopconductor{}, 195 | atom: &noopatom{}, 196 | }, 197 | true, 198 | }, 199 | { 200 | "invalid instance / nil atom", 201 | instance{ 202 | electron: noopelectron, 203 | conductor: &noopconductor{}, 204 | }, 205 | false, 206 | }, 207 | { 208 | "invalid instance / invalid electron", 209 | instance{ 210 | electron: &Electron{ 211 | SenderID: "empty", 212 | ID: "empty", 213 | }, 214 | conductor: &noopconductor{}, 215 | atom: &noopatom{}, 216 | }, 217 | false, 218 | }, 219 | { 220 | "invalid instance / invalid electron", 221 | instance{ 222 | electron: &Electron{}, 223 | conductor: &noopconductor{}, 224 | atom: &noopatom{}, 225 | }, 226 | false, 227 | }, 228 | { 229 | "invalid instance / invalid conductor", 230 | instance{ 231 | electron: noopelectron, 232 | atom: &noopatom{}, 233 | }, 234 | false, 235 | }, 236 | } 237 | 238 | for _, test := range tests { 239 | t.Run(test.name, func(t *testing.T) { 240 | valid := test.inst.Validate() 241 | if valid != test.valid { 242 | t.Fatalf( 243 | "valid mismatch, expected %v got %v", 244 | valid, 245 | test.inst.Validate(), 246 | ) 247 | } 248 | }) 249 | } 250 | } 251 | -------------------------------------------------------------------------------- /properties.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "bytes" 10 | "encoding/json" 11 | "errors" 12 | "time" 13 | ) 14 | 15 | // TODO: Set it up so that requests can be made to check the properties of 16 | // a bonded electron / atom at runtime 17 | 18 | // Properties is the struct for storing properties information after the 19 | // processing of an atom has completed so that it can be sent to the 20 | // original requestor 21 | type Properties struct { 22 | ElectronID string 23 | AtomID string 24 | Start time.Time 25 | End time.Time 26 | Error error 27 | Result []byte 28 | } 29 | 30 | // UnmarshalJSON reads in a []byte of JSON data and maps it to the Properties 31 | // struct properly for use throughout Atomizer 32 | func (p *Properties) UnmarshalJSON(data []byte) error { 33 | jsonP := struct { 34 | ElectronID string `json:"electronId"` 35 | AtomID string `json:"atomId"` 36 | Start time.Time `json:"starttime"` 37 | End time.Time `json:"endtime"` 38 | Error []byte `json:"error,omitempty"` 39 | Result json.RawMessage `json:"result"` 40 | }{} 41 | 42 | err := json.Unmarshal(data, &jsonP) 43 | if err != nil { 44 | return err 45 | } 46 | 47 | if jsonP.Error != nil { 48 | e := &Error{} 49 | err := json.Unmarshal(jsonP.Error, &e) 50 | if err == nil { 51 | p.Error = e 52 | } else { 53 | p.Error = errors.New(string(jsonP.Error)) 54 | } 55 | } 56 | 57 | p.ElectronID = jsonP.ElectronID 58 | p.AtomID = jsonP.AtomID 59 | p.Start = jsonP.Start 60 | p.End = jsonP.End 61 | p.Result = []byte(jsonP.Result) 62 | 63 | return nil 64 | } 65 | 66 | // MarshalJSON implements the custom json marshaler for properties 67 | func (p *Properties) MarshalJSON() ([]byte, error) { 68 | var eString []byte 69 | if p.Error != nil { 70 | _, ok := p.Error.(*Error) 71 | if ok { 72 | var err error 73 | eString, err = json.Marshal(p.Error) 74 | if err != nil { 75 | eString = []byte(p.Error.Error()) 76 | } 77 | } else { 78 | eString = []byte(p.Error.Error()) 79 | } 80 | } 81 | 82 | return json.Marshal(&struct { 83 | ElectronID string `json:"electronId"` 84 | AtomID string `json:"atomId"` 85 | Start time.Time `json:"starttime"` 86 | End time.Time `json:"endtime"` 87 | Error []byte `json:"error,omitempty"` 88 | Result json.RawMessage `json:"result"` 89 | }{ 90 | ElectronID: p.ElectronID, 91 | AtomID: p.AtomID, 92 | Start: p.Start, 93 | End: p.End, 94 | Error: eString, 95 | Result: json.RawMessage(p.Result), 96 | }) 97 | } 98 | 99 | // Equal determines if two properties structs are equal to eachother 100 | // TODO: Should this use reflect.DeepEqual? 101 | func (p *Properties) Equal(p2 *Properties) bool { 102 | var eEquals bool 103 | if p.Error != nil { 104 | if p2.Error != nil { 105 | eEquals = p.Error.Error() == p2.Error.Error() 106 | } 107 | } else if p.Error == nil && p2.Error == nil { 108 | eEquals = true 109 | } 110 | 111 | return p.ElectronID == p2.ElectronID && 112 | p.AtomID == p2.AtomID && 113 | p.Start.Equal(p2.Start) && 114 | p.End.Equal(p2.End) && 115 | bytes.Equal(p.Result, p2.Result) && 116 | eEquals 117 | } 118 | -------------------------------------------------------------------------------- /properties_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | "time" 7 | 8 | "github.com/davecgh/go-spew/spew" 9 | "github.com/google/go-cmp/cmp" 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | var nooppropNob64ErrJSON = `{"electronId":"test","atomId":"test","starttime":"0001-01-01T00:00:00Z","endtime":"0001-01-01T00:00:00Z","error":"no matchy","result":{"result":"test"}}` 14 | 15 | var nooppropJSON = `{"electronId":"test","atomId":"test","starttime":"0001-01-01T00:00:00Z","endtime":"0001-01-01T00:00:00Z","result":{"result":"test"}}` 16 | 17 | var noopprop = &Properties{ 18 | ElectronID: "test", 19 | AtomID: "test", 20 | Start: time.Time{}, 21 | End: time.Time{}, 22 | Error: nil, 23 | Result: []byte(`{"result":"test"}`), 24 | } 25 | 26 | var nooppropErrJSON = `{"electronId":"test","atomId":"test","starttime":"0001-01-01T00:00:00Z","endtime":"0001-01-01T00:00:00Z","error":"eyJldmVudCI6eyJtZXNzYWdlIjoidGVzdCIsImVsZWN0cm9uSUQiOiIiLCJhdG9tSUQiOiIiLCJjb25kdWN0b3JJRCI6IiJ9LCJpbnRlcm5hbCI6bnVsbH0=","result":{"result":"test"}}` 27 | 28 | var nooppropNoMatchErrJSON = `{"electronId":"test","atomId":"test","starttime":"0001-01-01T00:00:00Z","endtime":"0001-01-01T00:00:00Z","error":"eyJub21hdGNoIjoibm9tYXRjaCJ9","result":{"result":"test"}}` 29 | 30 | var nooppropErr = &Properties{ 31 | ElectronID: "test", 32 | AtomID: "test", 33 | Start: time.Time{}, 34 | End: time.Time{}, 35 | Error: simple("test", nil), 36 | Result: []byte(`{"result":"test"}`), 37 | } 38 | 39 | var nooppropNonAtomErrJSON = `{"electronId":"test","atomId":"test","starttime":"0001-01-01T00:00:00Z","endtime":"0001-01-01T00:00:00Z","error":"dGVzdA==","result":{"result":"test"}}` 40 | 41 | var nooppropNonAtomNoMatchErrJSON = `{"electronId":"test","atomId":"test","starttime":"0001-01-01T00:00:00Z","endtime":"0001-01-01T00:00:00Z","error":"eyJub21hdGNoIjoibm9tYXRjaCJ9","result":{"result":"test"}}` 42 | 43 | var nooppropNonAtomErr = &Properties{ 44 | ElectronID: "test", 45 | AtomID: "test", 46 | Start: time.Time{}, 47 | End: time.Time{}, 48 | Error: errors.New("test"), 49 | Result: []byte(`{"result":"test"}`), 50 | } 51 | 52 | func TestProperties_MarshalJSON(t *testing.T) { 53 | tests := []struct { 54 | name string 55 | p *Properties 56 | json string 57 | err bool 58 | }{ 59 | { 60 | "valid Properties", 61 | noopprop, 62 | nooppropJSON, 63 | false, 64 | }, 65 | { 66 | "valid Properties w/ error", 67 | nooppropErr, 68 | nooppropErrJSON, 69 | false, 70 | }, 71 | { 72 | "valid Properties w/ non-Atom error", 73 | nooppropNonAtomErr, 74 | nooppropNonAtomErrJSON, 75 | false, 76 | }, 77 | } 78 | 79 | for _, test := range tests { 80 | t.Run(test.name, func(t *testing.T) { 81 | res, err := json.Marshal(test.p) 82 | if err != nil && !test.err { 83 | t.Fatalf("expected success, got error | %s", err.Error()) 84 | return 85 | } 86 | 87 | if err == nil && test.err { 88 | t.Fatal("expected error") 89 | return 90 | } 91 | 92 | diff := cmp.Diff(test.json, string(res)) 93 | if diff != "" { 94 | t.Fatalf( 95 | "expected equality %s", 96 | diff, 97 | ) 98 | } 99 | }) 100 | } 101 | } 102 | 103 | func TestProperties_UnmarshalJSON(t *testing.T) { 104 | tests := []struct { 105 | name string 106 | p *Properties 107 | json string 108 | equal bool 109 | err bool 110 | }{ 111 | { 112 | "valid Properties", 113 | noopprop, 114 | nooppropJSON, 115 | true, 116 | false, 117 | }, 118 | { 119 | "valid Properties w/ error", 120 | nooppropErr, 121 | nooppropErrJSON, 122 | true, 123 | false, 124 | }, 125 | { 126 | "valid Properties w/ non-matching Atom error", 127 | nooppropErr, 128 | nooppropNoMatchErrJSON, 129 | false, 130 | false, 131 | }, 132 | { 133 | "valid Properties w/ non-Atom error", 134 | nooppropNonAtomErr, 135 | nooppropNonAtomErrJSON, 136 | true, 137 | false, 138 | }, 139 | { 140 | "valid Properties w/ non-matching non-Atom error", 141 | nooppropNonAtomErr, 142 | nooppropNonAtomNoMatchErrJSON, 143 | false, 144 | false, 145 | }, 146 | { 147 | "invalid Properties missing base64 encoding for err", 148 | nooppropErr, 149 | nooppropNob64ErrJSON, 150 | false, 151 | true, 152 | }, 153 | { 154 | "invalid json blob", 155 | &Properties{}, 156 | `{"empty"}`, 157 | false, 158 | true, 159 | }, 160 | } 161 | 162 | for _, test := range tests { 163 | t.Run(test.name, func(t *testing.T) { 164 | p := &Properties{} 165 | err := json.Unmarshal([]byte(test.json), &p) 166 | 167 | if err != nil && !test.err { 168 | t.Fatalf("expected success, got error | %s", err.Error()) 169 | return 170 | } 171 | 172 | if err == nil && test.err { 173 | t.Fatal("expected error") 174 | return 175 | } 176 | 177 | if !test.err { 178 | if cmp.Equal(test.p, p) != test.equal { 179 | t.Fatalf( 180 | "expected equality e[%s] != r[%s]", 181 | spew.Sdump(test.p), 182 | spew.Sdump(p), 183 | ) 184 | } 185 | } 186 | }) 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /register.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2019 Developer Network, LLC 2 | // 3 | // This file is subject to the terms and conditions defined in 4 | // file 'LICENSE', which is part of this source code package. 5 | 6 | package engine 7 | 8 | import ( 9 | "fmt" 10 | "sync" 11 | 12 | "go.devnw.com/validator" 13 | ) 14 | 15 | var registrant sync.Map 16 | 17 | // Registrations returns a channel which contains the init pre-registrations 18 | // for use by the atomizer 19 | func Registrations() []interface{} { 20 | registrations := make([]interface{}, 0) 21 | 22 | registrant.Range(func(key, value interface{}) bool { 23 | registrations = append(registrations, value) 24 | return true 25 | }) 26 | return registrations 27 | } 28 | 29 | // Register adds entries of different types that are used by the atomizer 30 | // and allows them to be pre-registered using an init script rather than 31 | // having them passed in later at run time. This is useful for some situations 32 | // where the user may not want to register explicitly 33 | func Register(values ...interface{}) error { 34 | for _, value := range values { 35 | // Validate the value coming into the register method 36 | if !validator.Valid(value) { 37 | return simple( 38 | fmt.Sprintf( 39 | "Invalid registration %s", 40 | ID(value)), 41 | nil, 42 | ) 43 | } 44 | 45 | // Type assert the value to ensure we're only 46 | // registering expected values in the maps 47 | switch value.(type) { 48 | case Conductor, Atom: 49 | // Registrations using the same key will 50 | // be overridden 51 | registrant.Store(ID(value), value) 52 | default: 53 | return simple( 54 | fmt.Sprintf( 55 | "unsupported type %s", 56 | ID(value)), 57 | nil, 58 | ) 59 | } 60 | } 61 | 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /register_mock_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "go.devnw.com/alog" 10 | ) 11 | 12 | func reset(ctx context.Context, t *testing.T) { 13 | registrant = sync.Map{} 14 | 15 | if ctx != nil { 16 | _ = alog.Global( 17 | ctx, // Default context 18 | "", // No prefix 19 | alog.DEFAULTTIMEFORMAT, // Standard time format 20 | time.UTC, // UTC logging 21 | alog.DEFAULTBUFFER, // Default buffer of 100 logs 22 | alog.TestDestinations(ctx, t)..., // Default destinations 23 | ) 24 | } 25 | } 26 | 27 | func resetB() { 28 | registrant = sync.Map{} 29 | 30 | _ = alog.Global( 31 | context.Background(), // Default context 32 | "", // No prefix 33 | alog.DEFAULTTIMEFORMAT, // Standard time format 34 | time.UTC, // UTC logging 35 | alog.DEFAULTBUFFER, // Default buffer of 100 logs 36 | alog.BenchDestinations()..., // Default destinations 37 | ) 38 | } 39 | -------------------------------------------------------------------------------- /register_test.go: -------------------------------------------------------------------------------- 1 | package engine 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | ) 7 | 8 | type invalidTestStruct struct{} 9 | 10 | func TestRegister(t *testing.T) { 11 | ctx, cancel := _ctx(context.TODO()) 12 | defer cancel() 13 | 14 | tests := []struct { 15 | name string 16 | key string 17 | value interface{} 18 | err bool 19 | }{ 20 | { 21 | "valid conductor registration", 22 | ID(noopconductor{}), 23 | &noopconductor{}, 24 | false, 25 | }, 26 | { 27 | "valid atom registration", 28 | ID(noopatom{}), 29 | &noopatom{}, 30 | false, 31 | }, 32 | { 33 | "invalid nil registration", 34 | "", 35 | nil, 36 | true, 37 | }, 38 | { 39 | "invalid interface registration", 40 | ID(invalidTestStruct{}), 41 | invalidTestStruct{}, 42 | true, 43 | }, 44 | } 45 | 46 | for _, test := range tests { 47 | t.Run(test.name, func(t *testing.T) { 48 | reset(ctx, t) 49 | defer reset(context.TODO(), t) 50 | 51 | err := Register(test.value) 52 | if err != nil && !test.err { 53 | t.Fatal(err) 54 | } 55 | 56 | _, ok := registrant.Load(test.key) 57 | if !ok && !test.err { 58 | t.Fatalf( 59 | "Test key [%s] failed to load", 60 | test.key, 61 | ) 62 | } 63 | }) 64 | } 65 | } 66 | --------------------------------------------------------------------------------